Bug 703266 Mirror mozbase to mozilla-central for peptest r=jhammel

testing/mozbase/Makefile.in

@@ -0,0 +1,73 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozbase.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Andrew Halberstadt <halbersa@gmail.com> (Original author)
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either of the GNU General Public License Version 2 or later (the "GPL"),
+# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+DEPTH = ../..
+topsrcdir = @top_srcdir@
+srcdir = @srcdir@
+VPATH = @srcdir@
+
+include $(DEPTH)/config/autoconf.mk
+
+MODULE = testing_mozbase
+
+include $(topsrcdir)/config/rules.mk
+
+# Harness packages from the srcdir;
+# python packages to be installed IN INSTALLATION ORDER.
+# Packages later in the list can depend only on packages earlier in the list.
+MOZBASE_PACKAGES = \
+  manifestdestiny \
+  mozdevice \
+  mozhttpd \
+  mozinfo \
+  mozinstall \
+  mozlog \
+  mozprocess \
+  mozprofile \
+  mozrunner \
+  $(NULL)
+
+MOZBASE_EXTRAS = \
+  setup_development.py \
+  README \
+  $(NULL)
+
+stage-package: PKG_STAGE = $(DIST)/test-package-stage
+stage-package:
+	$(NSINSTALL) -D $(PKG_STAGE)/mozbase
+	@(cd $(srcdir) && tar $(TAR_CREATE_FLAGS) - $(MOZBASE_PACKAGES)) | (cd $(PKG_STAGE)/mozbase && tar -xf -)
+	@(cd $(srcdir) && tar $(TAR_CREATE_FLAGS) - $(MOZBASE_EXTRAS)) | (cd $(PKG_STAGE)/mozbase && tar -xf -)

testing/mozbase/README

@@ -0,0 +1,7 @@
+This is the git repo for the mozbase suite of python utilities.
+
+Learn more about mozbase here: https://wiki.mozilla.org/Auto-tools/Projects/MozBase
+
+Bugs live at https://bugzilla.mozilla.org/buglist.cgi?resolution=---&component=Mozbase&product=Testing and https://bugzilla.mozilla.org/buglist.cgi?resolution=---&status_whiteboard_type=allwordssubstr&query_format=advanced&status_whiteboard=mozbase
+
+To file a bug, go to https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=Mozbase

testing/mozbase/manifestdestiny/README.txt

@@ -0,0 +1,345 @@
+ManifestDestiny
+===============
+
+Universal manifests for Mozilla test harnesses
+
+
+What is ManifestDestiny?
+------------------------
+
+What ManifestDestiny gives you::
+
+* manifests are (ordered) lists of tests
+* tests may have an arbitrary number of key, value pairs
+* the parser returns an ordered list of test data structures, which
+  are just dicts with some keys.  For example, a test with no
+  user-specified metadata looks like this::
+
+  [{'path':
+    '/home/jhammel/mozmill/src/ManifestDestiny/manifestdestiny/tests/testToolbar/testBackForwardButtons.js',
+    'name': 'testToolbar/testBackForwardButtons.js', 'here':
+    '/home/jhammel/mozmill/src/ManifestDestiny/manifestdestiny/tests',
+    'manifest': '/home/jhammel/mozmill/src/ManifestDestiny/manifestdestiny/tests',}]
+
+The keys displayed here (path, name, here, and manifest) are reserved
+keys for ManifestDestiny and any consuming APIs.  You can add
+additional key, value metadata to each test.
+
+
+Why have test manifests?
+------------------------
+
+Most Mozilla test harnesses work by crawling a directory structure.
+While this is straight-forward, manifests offer several practical
+advantages::
+
+* ability to turn a test off easily: if a test is broken on m-c
+  currently, the only way to turn it off, generally speaking, is just
+  removing the test.  Often this is undesirable, as if the test should
+  be dismissed because other people want to land and it can't be
+  investigated in real time (is it a failure? is the test bad? is no
+  one around that knows the test?), then backing out a test is at best
+  problematic.  With a manifest, a test may be disabled without
+  removing it from the tree and a bug filed with the appropriate
+  reason::
+
+   [test_broken.js]
+   disabled = https://bugzilla.mozilla.org/show_bug.cgi?id=123456
+
+* ability to run different (subsets of) tests on different
+  platforms. Traditionally, we've done a bit of magic or had the test
+  know what platform it would or would not run on. With manifests, you
+  can mark what platforms a test will or will not run on and change
+  these without changing the test.
+
+   [test_works_on_windows_only.js]
+   run-if = os == 'win'
+
+* ability to markup tests with metadata. We have a large, complicated,
+  and always changing infrastructure.  key, value metadata may be used
+  as an annotation to a test and appropriately curated and mined.  For
+  instance, we could mark certain tests as randomorange with a bug
+  number, if it were desirable.
+
+* ability to have sane and well-defined test-runs. You can keep
+  different manifests for different test runs and ``[include:]``
+  (sub)manifests as appropriate to your needs.
+
+
+Manifest Format
+---------------
+
+Manifests are .ini file with the section names denoting the path
+relative to the manifest::
+
+ [foo.js]
+ [bar.js]
+ [fleem.js]
+
+The sections are read in order. In addition, tests may include
+arbitrary key, value metadata to be used by the harness.  You may also
+have a ``[DEFAULT]`` section that will give key, value pairs that will
+be inherited by each test unless overridden::
+
+ [DEFAULT]
+ type = restart
+
+ [lilies.js]
+ color = white
+
+ [daffodils.js]
+ color = yellow
+ type = other
+ # override type from DEFAULT
+
+ [roses.js]
+ color = red
+
+You can also include other manifests::
+
+ [include:subdir/anothermanifest.ini]
+
+Manifests are included relative to the directory of the manifest with
+the ``[include:]`` directive unless they are absolute paths.
+
+
+Data
+----
+
+Manifest Destiny gives tests as a list of dictionaries (in python
+terms). 
+
+* path: full path to the test
+* name: short name of the test; this is the (usually) relative path
+  specified in the section name
+* here: the parent directory of the manifest
+* manifest: the path to the manifest containing the test
+
+This data corresponds to a one-line manifest::
+
+ [testToolbar/testBackForwardButtons.js]
+
+If additional key, values were specified, they would be in this dict
+as well.
+
+Outside of the reserved keys, the remaining key, values
+are up to convention to use.  There is a (currently very minimal)
+generic integration layer in ManifestDestiny for use of all harnesses,
+``manifestparser.TestManifest``.
+For instance, if the 'disabled' key is present, you can get the set of
+tests without disabled (various other queries are doable as well).
+
+Since the system is convention-based, the harnesses may do whatever
+they want with the data.  They may ignore it completely, they may use
+the provided integration layer, or they may provide their own
+integration layer.  This should allow whatever sort of logic is
+desired.  For instance, if in yourtestharness you wanted to run only on
+mondays for a certain class of tests::
+
+ tests = []
+ for test in manifests.tests:
+     if 'runOnDay' in test:
+        if calendar.day_name[calendar.weekday(*datetime.datetime.now().timetuple()[:3])].lower() == test['runOnDay'].lower():
+            tests.append(test)
+     else:
+        tests.append(test)
+
+To recap:
+* the manifests allow you to specify test data
+* the parser gives you this data
+* you can use it however you want or process it further as you need
+
+Tests are denoted by sections in an .ini file (see
+http://hg.mozilla.org/automation/ManifestDestiny/file/tip/manifestdestiny/tests/mozmill-example.ini). 
+
+Additional manifest files may be included with an ``[include:]`` directive::
+
+ [include:path-to-additional-file.manifest]
+
+The path to included files is relative to the current manifest.
+
+The ``[DEFAULT]`` section contains variables that all tests inherit from.
+
+Included files will inherit the top-level variables but may override
+in their own ``[DEFAULT]`` section.
+
+
+ManifestDestiny Architecture
+----------------------------
+
+There is a two- or three-layered approach to the ManifestDestiny
+architecture, depending on your needs::
+
+1. ManifestParser: this is a generic parser for .ini manifests that
+facilitates the `[include:]` logic and the inheritence of
+metadata. Despite the internal variable being called ``self.tests``
+(an oversight), this layer has nothing in particular to do with tests.
+
+2. TestManifest: this is a harness-agnostic integration layer that is
+test-specific. TestManifest faciliates ``skip-if`` and ``run-if``
+logic.
+
+3. Optionally, a harness will have an integration layer than inherits
+from TestManifest if more harness-specific customization is desired at
+the manifest level.
+
+See the source code at http://hg.mozilla.org/automation/ManifestDestiny
+and
+http://hg.mozilla.org/automation/ManifestDestiny/file/tip/manifestparser.py
+in particular.
+
+
+Using Manifests
+---------------
+
+A test harness will normally call ``TestManifest.active_tests`` (
+http://hg.mozilla.org/automation/ManifestDestiny/file/c0399fbfa830/manifestparser.py#l506 )::
+
+   506     def active_tests(self, exists=True, disabled=True, **tags):
+
+The manifests are passed to the ``__init__`` or ``read`` methods with
+appropriate arguments.  ``active_tests`` then allows you to select the
+tests you want::
+
+- exists : return only existing tests
+- disabled : whether to return disabled tests; if not these will be
+  filtered out; if True (the default), the ``disabled`` key of a
+  test's metadata will be present and will be set to the reason that a
+  test is disabled
+- tags : keys and values to filter on (e.g. ``os='linux'``)
+
+``active_tests`` looks for tests with ``skip-if.${TAG}`` or
+``run-if``.  If the condition is or is not fulfilled,
+respectively, the test is marked as disabled.  For instance, if you
+pass ``**dict(os='linux')`` as ``**tags``, if a test contains a line
+``skip-if = os == 'linux'`` this test will be disabled, or 
+``run-if = os = 'win'`` in which case the test will also be disabled.  It
+is up to the harness to pass in tags appropriate to its usage.  
+
+
+Creating Manifests
+------------------
+
+ManifestDestiny comes with a console script, ``manifestparser create``, that
+may be used to create a seed manifest structure from a directory of
+files.  Run ``manifestparser help create`` for usage information.
+
+
+Copying Manifests
+-----------------
+
+To copy tests and manifests from a source::
+
+ manifestparser [options] copy from_manifest to_directory -tag1 -tag2 --key1=value1 key2=value2 ...
+
+
+Upating Tests
+-------------
+
+To update the tests associated with with a manifest from a source
+directory::
+
+ manifestparser [options] update manifest from_directory -tag1 -tag2 --key1=value1 --key2=value2 ...
+
+
+Tests
+-----
+
+ManifestDestiny includes a suite of tests:
+
+http://hg.mozilla.org/automation/ManifestDestiny/file/tip/manifestdestiny/tests
+
+``test_manifest.txt`` is a doctest that may be helpful in figuring out
+how to use the API.  Tests are run via ``python test.py``.
+
+
+Bugs
+----
+
+Please file any bugs or feature requests at 
+
+https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=ManifestParser
+
+Or contact jhammel @mozilla.org or in #ateam on irc.mozilla.org
+
+
+CLI
+---
+
+Run ``manifestparser help`` for usage information.
+
+To create a manifest from a set of directories::
+
+ manifestparser [options] create directory <directory> <...> [create-options]
+
+To output a manifest of tests::
+
+ manifestparser [options] write manifest <manifest> <...> -tag1 -tag2 --key1=value1 --key2=value2 ...
+
+To copy tests and manifests from a source::
+
+ manifestparser [options] copy from_manifest to_manifest -tag1 -tag2 --key1=value1 key2=value2 ...
+
+To update the tests associated with with a manifest from a source
+directory::
+
+ manifestparser [options] update manifest from_directory -tag1 -tag2 --key1=value1 --key2=value2 ...
+
+
+Design Considerations
+---------------------
+
+Contrary to some opinion, manifestparser.py and the associated .ini
+format were not magically plucked from the sky but were descended upon
+through several design considerations.
+
+* test manifests should be ordered.  While python 2.6 and greater has
+  a ConfigParser that can use an ordered dictionary, it is a
+  requirement that we support python 2.4 for the build + testing
+  environment.  To that end, a ``read_ini`` function was implemented
+  in manifestparser.py that should be the equivalent of the .ini
+  dialect used by ConfigParser.
+
+* the manifest format should be easily human readable/writable.  While
+  there was initially some thought of using JSON, there was pushback
+  that JSON was not easily editable.  An ideal manifest format would
+  degenerate to a line-separated list of files.  While .ini format
+  requires an additional ``[]`` per line, and while there have been
+  complaints about this, hopefully this is good enough.
+
+* python does not have an in-built YAML parser.  Since it was
+  undesirable for manifestparser.py to have any dependencies, YAML was
+  dismissed as a format.
+
+* we could have used a proprietary format but decided against it.
+  Everyone knows .ini and there are good tools to deal with it.
+  However, since read_ini is the only function that transforms a
+  manifest to a list of key, value pairs, while the implications for
+  changing the format impacts downstream code, doing so should be
+  programmatically simple.
+
+* there should be a single file that may easily be
+  transported. Traditionally, test harnesses have lived in
+  mozilla-central. This is less true these days and it is increasingly
+  likely that more tests will not live in mozilla-central going
+  forward.  So ``manifestparser.py`` should be highly consumable. To
+  this end, it is a single file, as appropriate to mozilla-central,
+  which is also a working python package deployed to PyPI for easy
+  installation. 
+
+
+Historical Reference
+--------------------
+
+Date-ordered list of links about how manifests came to be where they are today::
+
+* https://wiki.mozilla.org/Auto-tools/Projects/UniversalManifest
+* http://alice.nodelman.net/blog/post/2010/05/
+* http://alice.nodelman.net/blog/post/universal-manifest-for-unit-tests-a-proposal/
+* https://elvis314.wordpress.com/2010/07/05/improving-personal-hygiene-by-adjusting-mochitests/
+* https://elvis314.wordpress.com/2010/07/27/types-of-data-we-care-about-in-a-manifest/
+* https://bugzilla.mozilla.org/show_bug.cgi?id=585106
+* http://elvis314.wordpress.com/2011/05/20/converting-xpcshell-from-listing-directories-to-a-manifest/
+* https://bugzilla.mozilla.org/show_bug.cgi?id=616999
+* https://wiki.mozilla.org/Auto-tools/Projects/ManifestDestiny
+* https://developer.mozilla.org/en/Writing_xpcshell-based_unit_tests#Adding_your_tests_to_the_xpcshell_manifest

testing/mozbase/manifestdestiny/setup.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is manifestdestiny.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#     Jeff Hammel <jhammel@mozilla.com>     (Original author)
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either of the GNU General Public License Version 2 or later (the "GPL"),
+# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+# The real details are in manifestparser.py; this is just a front-end
+# BUT use this file when you want to distribute to python!
+# otherwise setuptools will complain that it can't find setup.py
+# and result in a useless package
+
+import sys
+from manifestparser import SetupCLI
+SetupCLI(None)(None, sys.argv[1:])

testing/mozbase/manifestdestiny/tests/filter-example.ini

@@ -0,0 +1,11 @@
+# illustrate test filters based on various categories
+
+[windowstest]
+run-if = os == 'win'
+
+[fleem]
+skip-if = os == 'mac'
+
+[linuxtest]
+skip-if = (os == 'mac') || (os == 'win')
+fail-if = toolkit == 'cocoa'

testing/mozbase/manifestdestiny/tests/fleem

@@ -0,0 +1 @@
+# dummy spot for "fleem" test

testing/mozbase/manifestdestiny/tests/include-example.ini

@@ -0,0 +1,11 @@
+[DEFAULT]
+foo = bar
+
+[include:include/bar.ini]
+
+[fleem]
+
+[include:include/foo.ini]
+red = roses
+blue = violets
+yellow = daffodils

testing/mozbase/manifestdestiny/tests/include/bar.ini

@@ -0,0 +1,4 @@
+[DEFAULT]
+foo = fleem
+
+[crash-handling]

testing/mozbase/manifestdestiny/tests/include/crash-handling

@@ -0,0 +1 @@
+# dummy spot for "crash-handling" test

testing/mozbase/manifestdestiny/tests/include/flowers

@@ -0,0 +1 @@
+# dummy spot for "flowers" test

testing/mozbase/manifestdestiny/tests/include/foo.ini

@@ -0,0 +1,5 @@
+[DEFAULT]
+blue = ocean
+
+[flowers]
+yellow = submarine

testing/mozbase/manifestdestiny/tests/mozmill-example.ini

@@ -0,0 +1,80 @@
+[testAddons/testDisableEnablePlugin.js]
+[testAddons/testGetAddons.js]
+[testAddons/testSearchAddons.js]
+[testAwesomeBar/testAccessLocationBar.js]
+[testAwesomeBar/testCheckItemHighlight.js]
+[testAwesomeBar/testEscapeAutocomplete.js]
+[testAwesomeBar/testFaviconInAutocomplete.js]
+[testAwesomeBar/testGoButton.js]
+[testAwesomeBar/testLocationBarSearches.js]
+[testAwesomeBar/testPasteLocationBar.js]
+[testAwesomeBar/testSuggestHistoryBookmarks.js]
+[testAwesomeBar/testVisibleItemsMax.js]
+[testBookmarks/testAddBookmarkToMenu.js]
+[testCookies/testDisableCookies.js]
+[testCookies/testEnableCookies.js]
+[testCookies/testRemoveAllCookies.js]
+[testCookies/testRemoveCookie.js]
+[testDownloading/testCloseDownloadManager.js]
+[testDownloading/testDownloadStates.js]
+[testDownloading/testOpenDownloadManager.js]
+[testFindInPage/testFindInPage.js]
+[testFormManager/testAutoCompleteOff.js]
+[testFormManager/testBasicFormCompletion.js]
+[testFormManager/testClearFormHistory.js]
+[testFormManager/testDisableFormManager.js]
+[testGeneral/testGoogleSuggestions.js]
+[testGeneral/testStopReloadButtons.js]
+[testInstallation/testBreakpadInstalled.js]
+[testLayout/testNavigateFTP.js]
+[testPasswordManager/testPasswordNotSaved.js]
+[testPasswordManager/testPasswordSavedAndDeleted.js]
+[testPopups/testPopupsAllowed.js]
+[testPopups/testPopupsBlocked.js]
+[testPreferences/testPaneRetention.js]
+[testPreferences/testPreferredLanguage.js]
+[testPreferences/testRestoreHomepageToDefault.js]
+[testPreferences/testSetToCurrentPage.js]
+[testPreferences/testSwitchPanes.js]
+[testPrivateBrowsing/testAboutPrivateBrowsing.js]
+[testPrivateBrowsing/testCloseWindow.js]
+[testPrivateBrowsing/testDisabledElements.js]
+[testPrivateBrowsing/testDisabledPermissions.js]
+[testPrivateBrowsing/testDownloadManagerClosed.js]
+[testPrivateBrowsing/testGeolocation.js]
+[testPrivateBrowsing/testStartStopPBMode.js]
+[testPrivateBrowsing/testTabRestoration.js]
+[testPrivateBrowsing/testTabsDismissedOnStop.js]
+[testSearch/testAddMozSearchProvider.js]
+[testSearch/testFocusAndSearch.js]
+[testSearch/testGetMoreSearchEngines.js]
+[testSearch/testOpenSearchAutodiscovery.js]
+[testSearch/testRemoveSearchEngine.js]
+[testSearch/testReorderSearchEngines.js]
+[testSearch/testRestoreDefaults.js]
+[testSearch/testSearchSelection.js]
+[testSearch/testSearchSuggestions.js]
+[testSecurity/testBlueLarry.js]
+[testSecurity/testDefaultPhishingEnabled.js]
+[testSecurity/testDefaultSecurityPrefs.js]
+[testSecurity/testEncryptedPageWarning.js]
+[testSecurity/testGreenLarry.js]
+[testSecurity/testGreyLarry.js]
+[testSecurity/testIdentityPopupOpenClose.js]
+[testSecurity/testSSLDisabledErrorPage.js]
+[testSecurity/testSafeBrowsingNotificationBar.js]
+[testSecurity/testSafeBrowsingWarningPages.js]
+[testSecurity/testSecurityInfoViaMoreInformation.js]
+[testSecurity/testSecurityNotification.js]
+[testSecurity/testSubmitUnencryptedInfoWarning.js]
+[testSecurity/testUnknownIssuer.js]
+[testSecurity/testUntrustedConnectionErrorPage.js]
+[testSessionStore/testUndoTabFromContextMenu.js]
+[testTabbedBrowsing/testBackgroundTabScrolling.js]
+[testTabbedBrowsing/testCloseTab.js]
+[testTabbedBrowsing/testNewTab.js]
+[testTabbedBrowsing/testNewWindow.js]
+[testTabbedBrowsing/testOpenInBackground.js]
+[testTabbedBrowsing/testOpenInForeground.js]
+[testTechnicalTools/testAccessPageInfoDialog.js]
+[testToolbar/testBackForwardButtons.js]

testing/mozbase/manifestdestiny/tests/mozmill-restart-example.ini

@@ -0,0 +1,26 @@
+[DEFAULT]
+type = restart
+
+[restartTests/testExtensionInstallUninstall/test2.js]
+foo = bar
+
+[restartTests/testExtensionInstallUninstall/test1.js]
+foo = baz
+
+[restartTests/testExtensionInstallUninstall/test3.js]
+[restartTests/testSoftwareUpdateAutoProxy/test2.js]
+[restartTests/testSoftwareUpdateAutoProxy/test1.js]
+[restartTests/testMasterPassword/test1.js]
+[restartTests/testExtensionInstallGetAddons/test2.js]
+[restartTests/testExtensionInstallGetAddons/test1.js]
+[restartTests/testMultipleExtensionInstallation/test2.js]
+[restartTests/testMultipleExtensionInstallation/test1.js]
+[restartTests/testThemeInstallUninstall/test2.js]
+[restartTests/testThemeInstallUninstall/test1.js]
+[restartTests/testThemeInstallUninstall/test3.js]
+[restartTests/testDefaultBookmarks/test1.js]
+[softwareUpdate/testFallbackUpdate/test2.js]
+[softwareUpdate/testFallbackUpdate/test1.js]
+[softwareUpdate/testFallbackUpdate/test3.js]
+[softwareUpdate/testDirectUpdate/test2.js]
+[softwareUpdate/testDirectUpdate/test1.js]

testing/mozbase/manifestdestiny/tests/path-example.ini

@@ -0,0 +1,2 @@
+[foo]
+path = fleem

testing/mozbase/manifestdestiny/tests/test.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+# 
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+# 
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+# 
+# The Original Code is mozilla.org code.
+# 
+# The Initial Developer of the Original Code is
+# Mozilla.org.
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#     Jeff Hammel <jhammel@mozilla.com>     (Original author)
+# 
+# Alternatively, the contents of this file may be used under the terms of
+# either of the GNU General Public License Version 2 or later (the "GPL"),
+# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+# 
+# ***** END LICENSE BLOCK *****
+
+"""tests for ManifestDestiny"""
+
+import doctest
+import os
+import sys
+from optparse import OptionParser
+
+def run_tests(raise_on_error=False, report_first=False):
+
+    # add results here
+    results = {}
+
+    # doctest arguments
+    directory = os.path.dirname(os.path.abspath(__file__))
+    extraglobs = {}
+    doctest_args = dict(extraglobs=extraglobs,
+                        module_relative=False,
+                        raise_on_error=raise_on_error)
+    if report_first:
+        doctest_args['optionflags'] = doctest.REPORT_ONLY_FIRST_FAILURE
+                                
+    # gather tests
+    directory = os.path.dirname(os.path.abspath(__file__))
+    tests =  [ test for test in os.listdir(directory)
+               if test.endswith('.txt') and test.startswith('test_')]
+    os.chdir(directory)
+
+    # run the tests
+    for test in tests:
+        try:
+            results[test] = doctest.testfile(test, **doctest_args)
+        except doctest.DocTestFailure, failure:
+            raise
+        except doctest.UnexpectedException, failure:
+            raise failure.exc_info[0], failure.exc_info[1], failure.exc_info[2]
+        
+    return results
+                                
+
+def main(args=sys.argv[1:]):
+
+    # parse command line options
+    parser = OptionParser(description=__doc__)
+    parser.add_option('--raise', dest='raise_on_error',
+                      default=False, action='store_true',
+                      help="raise on first error")
+    parser.add_option('--report-first', dest='report_first',
+                      default=False, action='store_true',
+                      help="report the first error only (all tests will still run)")
+    parser.add_option('-q', '--quiet', dest='quiet',
+                      default=False, action='store_true',
+                      help="minimize output")
+    options, args = parser.parse_args(args)
+    quiet = options.__dict__.pop('quiet')
+
+    # run the tests
+    results = run_tests(**options.__dict__)
+
+    # check for failure
+    failed = False
+    for result in results.values():
+        if result[0]: # failure count; http://docs.python.org/library/doctest.html#basic-api
+            failed = True
+            break
+    if failed:
+        sys.exit(1) # error
+    if not quiet:
+        # print results
+        print "manifestparser.py: All tests pass!"
+        for test in sorted(results.keys()):
+            result = results[test]
+            print "%s: failed=%s, attempted=%s" % (test, result[0], result[1])
+               
+if __name__ == '__main__':
+    main()

testing/mozbase/manifestdestiny/tests/test_expressionparser.txt

@@ -0,0 +1,120 @@
+Test Expressionparser
+=====================
+
+Test the conditional expression parser.
+
+Boilerplate::
+
+    >>> from manifestparser import parse
+
+Test basic values::
+
+    >>> parse("1")
+    1
+    >>> parse("100")
+    100
+    >>> parse("true")
+    True
+    >>> parse("false")
+    False
+    >>> '' == parse('""')
+    True
+    >>> parse('"foo bar"')
+    'foo bar'
+    >>> parse("'foo bar'")
+    'foo bar'
+    >>> parse("foo", foo=1)
+    1
+    >>> parse("bar", bar=True)
+    True
+    >>> parse("abc123", abc123="xyz")
+    'xyz'
+
+Test equality::
+
+    >>> parse("true == true")
+    True
+    >>> parse("false == false")
+    True
+    >>> parse("false == false")
+    True
+    >>> parse("1 == 1")
+    True
+    >>> parse("100 == 100")
+    True
+    >>> parse('"some text" == "some text"')
+    True
+    >>> parse("true != false")
+    True
+    >>> parse("1 != 2")
+    True
+    >>> parse('"text" != "other text"')
+    True
+    >>> parse("foo == true", foo=True)
+    True
+    >>> parse("foo == 1", foo=1)
+    True
+    >>> parse('foo == "bar"', foo='bar')
+    True
+    >>> parse("foo == bar", foo=True, bar=True)
+    True
+    >>> parse("true == foo", foo=True)
+    True
+    >>> parse("foo != true", foo=False)
+    True
+    >>> parse("foo != 2", foo=1)
+    True
+    >>> parse('foo != "bar"', foo='abc')
+    True
+    >>> parse("foo != bar", foo=True, bar=False)
+    True
+    >>> parse("true != foo", foo=False)
+    True
+    >>> parse("!false")
+    True
+
+Test conjunctions::
+    
+    >>> parse("true && true")
+    True
+    >>> parse("true || false")
+    True
+    >>> parse("false || false")
+    False
+    >>> parse("true && false")
+    False
+    >>> parse("true || false && false")
+    True
+
+Test parentheses::
+    
+    >>> parse("(true)")
+    True
+    >>> parse("(10)")
+    10
+    >>> parse('("foo")')
+    'foo'
+    >>> parse("(foo)", foo=1)
+    1
+    >>> parse("(true == true)")
+    True
+    >>> parse("(true != false)")
+    True
+    >>> parse("(true && true)")
+    True
+    >>> parse("(true || false)")
+    True
+    >>> parse("(true && true || false)")
+    True
+    >>> parse("(true || false) && false")
+    False
+    >>> parse("(true || false) && true")
+    True
+    >>> parse("true && (true || false)")
+    True
+    >>> parse("true && (true || false)")
+    True
+    >>> parse("(true && false) || (true && (true || false))")
+    True
+        
+

testing/mozbase/manifestdestiny/tests/test_manifestparser.txt

@@ -0,0 +1,217 @@
+Test the manifest parser
+========================
+
+You must have ManifestDestiny installed before running these tests.
+Run ``python manifestparser.py setup develop`` with setuptools installed.
+
+Ensure basic parser is sane::
+
+    >>> from manifestparser import ManifestParser
+    >>> parser = ManifestParser()
+    >>> parser.read('mozmill-example.ini')
+    >>> tests = parser.tests
+    >>> len(tests) == len(file('mozmill-example.ini').read().strip().splitlines())
+    True
+    
+Ensure that capitalization and order aren't an issue:
+
+    >>> lines = ['[%s]' % test['name'] for test in tests]
+    >>> lines == file('mozmill-example.ini').read().strip().splitlines()
+    True
+
+Show how you select subsets of tests:
+
+    >>> parser.read('mozmill-restart-example.ini')
+    >>> restart_tests = parser.get(type='restart')
+    >>> len(restart_tests) < len(parser.tests)
+    True
+    >>> import os
+    >>> len(restart_tests) == len(parser.get(manifest=os.path.abspath('mozmill-restart-example.ini')))
+    True
+    >>> assert not [test for test in restart_tests if test['manifest'] != os.path.abspath('mozmill-restart-example.ini')]
+    >>> parser.get('name', tags=['foo'])
+    ['restartTests/testExtensionInstallUninstall/test2.js', 'restartTests/testExtensionInstallUninstall/test1.js']
+    >>> parser.get('name', foo='bar')
+    ['restartTests/testExtensionInstallUninstall/test2.js']
+
+Illustrate how include works::
+
+    >>> parser = ManifestParser(manifests=('include-example.ini',))
+
+All of the tests should be included, in order::
+
+    >>> parser.get('name')
+    ['crash-handling', 'fleem', 'flowers']
+    >>> [(test['name'], os.path.basename(test['manifest'])) for test in parser.tests]
+    [('crash-handling', 'bar.ini'), ('fleem', 'include-example.ini'), ('flowers', 'foo.ini')]
+
+The manifests should be there too::
+
+    >>> len(parser.manifests())
+    3
+
+We're already in the root directory::
+
+    >>> os.getcwd() == parser.rootdir
+    True
+
+DEFAULT values should persist across includes, unless they're
+overwritten.  In this example, include-example.ini sets foo=bar, but
+its overridden to fleem in bar.ini::
+
+    >>> parser.get('name', foo='bar')
+    ['fleem', 'flowers']
+    >>> parser.get('name', foo='fleem')
+    ['crash-handling']
+
+Passing parameters in the include section allows defining variables in
+the submodule scope:
+
+    >>> parser.get('name', tags=['red'])
+    ['flowers']
+
+However, this should be overridable from the DEFAULT section in the
+included file and that overridable via the key directly connected to
+the test::
+
+    >>> parser.get(name='flowers')[0]['blue']
+    'ocean'
+    >>> parser.get(name='flowers')[0]['yellow']
+    'submarine'
+
+You can query multiple times if you need to::
+
+    >>> flowers = parser.get(foo='bar')
+    >>> len(flowers)
+    2
+    >>> roses = parser.get(tests=flowers, red='roses')
+
+Using the inverse flag should invert the set of tests returned::
+
+    >>> parser.get('name', inverse=True, tags=['red'])
+    ['crash-handling', 'fleem']
+
+All of the included tests actually exist::
+
+    >>> [i['name'] for i in parser.missing()]
+    []
+
+Write the output to a manifest:
+
+    >>> from StringIO import StringIO
+    >>> buffer = StringIO()
+    >>> parser.write(fp=buffer, global_kwargs={'foo': 'bar'})
+    >>> buffer.getvalue().strip()
+    '[DEFAULT]\nfoo = bar\n\n[fleem]\n\n[include/flowers]\nblue = ocean\nred = roses\nyellow = submarine'
+
+Test our ability to convert a static directory structure to a
+manifest. First, stub out a directory with files in it::
+
+    >>> import shutil, tempfile
+    >>> def create_stub():
+    ...     directory = tempfile.mkdtemp()
+    ...     for i in 'foo', 'bar', 'fleem':
+    ...         file(os.path.join(directory, i), 'w').write(i)
+    ...     subdir = os.path.join(directory, 'subdir')
+    ...     os.mkdir(subdir)
+    ...     file(os.path.join(subdir, 'subfile'), 'w').write('baz')
+    ...     return directory
+    >>> stub = create_stub()
+    >>> os.path.exists(stub) and os.path.isdir(stub)
+    True
+
+Make a manifest for it::
+
+    >>> from manifestparser import convert
+    >>> print convert([stub])
+    [bar]
+    [fleem]
+    [foo]
+    [subdir/subfile]
+    >>> shutil.rmtree(stub)
+
+Now do the same thing but keep the manifests in place::
+
+    >>> stub = create_stub()
+    >>> convert([stub], write='manifest.ini')
+    >>> sorted(os.listdir(stub))
+    ['bar', 'fleem', 'foo', 'manifest.ini', 'subdir']
+    >>> parser = ManifestParser()
+    >>> parser.read(os.path.join(stub, 'manifest.ini'))
+    >>> [i['name'] for i in parser.tests]
+    ['subfile', 'bar', 'fleem', 'foo']
+    >>> parser = ManifestParser()
+    >>> parser.read(os.path.join(stub, 'subdir', 'manifest.ini'))
+    >>> len(parser.tests)
+    1
+    >>> parser.tests[0]['name']
+    'subfile'
+    >>> shutil.rmtree(stub)
+
+Test our ability to copy a set of manifests::
+
+    >>> tempdir = tempfile.mkdtemp()
+    >>> manifest = ManifestParser(manifests=('include-example.ini',))
+    >>> manifest.copy(tempdir)
+    >>> sorted(os.listdir(tempdir))
+    ['fleem', 'include', 'include-example.ini']
+    >>> sorted(os.listdir(os.path.join(tempdir, 'include')))
+    ['bar.ini', 'crash-handling', 'flowers', 'foo.ini']
+    >>> from_manifest = ManifestParser(manifests=('include-example.ini',))
+    >>> to_manifest = os.path.join(tempdir, 'include-example.ini')
+    >>> to_manifest = ManifestParser(manifests=(to_manifest,))
+    >>> to_manifest.get('name') == from_manifest.get('name')
+    True
+    >>> shutil.rmtree(tempdir)
+
+Test our ability to update tests from a manifest and a directory of
+files::
+
+    >>> tempdir = tempfile.mkdtemp()
+    >>> for i in range(10):
+    ...     file(os.path.join(tempdir, str(i)), 'w').write(str(i))
+
+First, make a manifest::
+
+    >>> manifest = convert([tempdir])
+    >>> newtempdir = tempfile.mkdtemp()
+    >>> manifest_file = os.path.join(newtempdir, 'manifest.ini')
+    >>> file(manifest_file,'w').write(manifest)
+    >>> manifest = ManifestParser(manifests=(manifest_file,))
+    >>> manifest.get('name') == [str(i) for i in range(10)]
+    True
+
+All of the tests are initially missing::
+
+    >>> [i['name'] for i in manifest.missing()] == [str(i) for i in range(10)]
+    True
+
+But then we copy one over::
+
+    >>> manifest.get('name', name='1')
+    ['1']
+    >>> manifest.update(tempdir, name='1')
+    >>> sorted(os.listdir(newtempdir))
+    ['1', 'manifest.ini']
+
+Update that one file and copy all the "tests"::
+   
+    >>> file(os.path.join(tempdir, '1'), 'w').write('secret door')
+    >>> manifest.update(tempdir)
+    >>> sorted(os.listdir(newtempdir))
+    ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'manifest.ini']
+    >>> file(os.path.join(newtempdir, '1')).read().strip()
+    'secret door'
+
+Clean up::
+
+    >>> shutil.rmtree(tempdir)
+    >>> shutil.rmtree(newtempdir)
+
+You can override the path in the section too.  This shows that you can
+use a relative path::
+
+    >>> manifest = ManifestParser(manifests=('path-example.ini',))
+    >>> manifest.tests[0]['path'] == os.path.abspath('fleem')
+    True
+

testing/mozbase/manifestdestiny/tests/test_testmanifest.txt

@@ -0,0 +1,32 @@
+Test the Test Manifest
+======================
+
+Boilerplate::
+
+    >>> import os
+
+Test filtering based on platform::
+
+    >>> from manifestparser import TestManifest
+    >>> manifest = TestManifest(manifests=('filter-example.ini',))
+    >>> [i['name'] for i in manifest.active_tests(os='win', disabled=False, exists=False)]
+    ['windowstest', 'fleem']
+    >>> [i['name'] for i in manifest.active_tests(os='linux', disabled=False, exists=False)]
+    ['fleem', 'linuxtest']
+
+Look for existing tests.  There is only one::
+
+    >>> [i['name'] for i in manifest.active_tests()]
+    ['fleem']
+
+You should be able to expect failures::
+
+    >>> last_test = manifest.active_tests(exists=False, toolkit='gtk2')[-1]
+    >>> last_test['name']
+    'linuxtest'
+    >>> last_test['expected']
+    'pass'
+    >>> last_test = manifest.active_tests(exists=False, toolkit='cocoa')[-1]
+    >>> last_test['expected']
+    'fail'
+

testing/mozbase/mozhttpd/README.md

@@ -0,0 +1 @@
+basic python webserver, tested with talos

testing/mozbase/mozhttpd/mozhttpd.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# the Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Joel Maher <joel.maher@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import BaseHTTPServer
+import SimpleHTTPServer
+import threading
+import sys
+import os
+import urllib
+import re
+from SocketServer import ThreadingMixIn
+
+class EasyServer(ThreadingMixIn, BaseHTTPServer.HTTPServer):
+    allow_reuse_address = True
+    
+class MozRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    docroot = os.getcwd()
+
+    def parse_request(self):
+        retval = SimpleHTTPServer.SimpleHTTPRequestHandler.parse_request(self)
+        if '?' in self.path:
+            # ignore query string, otherwise SimpleHTTPRequestHandler 
+            # will treat it as PATH_INFO for `translate_path`
+            self.path = self.path.split('?', 1)[0]
+        return retval
+
+    def translate_path(self, path):
+        path = path.strip('/').split()
+        if path == ['']:
+            path = []
+        path.insert(0, self.docroot)
+        return os.path.join(*path)
+
+    # I found on my local network that calls to this were timing out
+    # I believe all of these calls are from log_message
+    def address_string(self):
+        return "a.b.c.d"
+
+    # This produces a LOT of noise
+    def log_message(self, format, *args):
+        pass
+
+class MozHttpd(object):
+
+    def __init__(self, host="127.0.0.1", port=8888, docroot=os.getcwd()):
+        self.host = host
+        self.port = int(port)
+        self.docroot = docroot
+        self.httpd = None
+
+    def start(self, block=False):
+        """
+        start the server.  If block is True, the call will not return.
+        If block is False, the server will be started on a separate thread that
+        can be terminated by a call to .stop()
+        """
+
+        class MozRequestHandlerInstance(MozRequestHandler):
+            docroot = self.docroot
+
+        self.httpd = EasyServer((self.host, self.port), MozRequestHandlerInstance)
+        if block:
+            self.httpd.serve_forever()
+        else:
+            self.server = threading.Thread(target=self.httpd.serve_forever)
+            self.server.setDaemon(True) # don't hang on exit
+            self.server.start()
+        
+    def testServer(self):
+        fileList = os.listdir(self.docroot)
+        filehandle = urllib.urlopen('http://%s:%s/?foo=bar&fleem=&foo=fleem' % (self.host, self.port))
+        data = filehandle.readlines()
+        filehandle.close()
+
+        retval = True
+
+        for line in data:
+            found = False
+            # '@' denotes a symlink and we need to ignore it.
+            webline = re.sub('\<[a-zA-Z0-9\-\_\.\=\"\'\/\\\%\!\@\#\$\^\&\*\(\) ]*\>', '', line.strip('\n')).strip('/').strip().strip('@')
+            if webline != "":
+                if webline == "Directory listing for":
+                    found = True
+                else:
+                    for fileName in fileList:
+                        if fileName == webline:
+                            found = True
+                
+                if not found:
+                    retval = False
+                    print >> sys.stderr, "NOT FOUND: " + webline.strip()
+        return retval
+
+    def stop(self):
+        if self.httpd:
+            self.httpd.shutdown()
+        self.httpd = None
+
+    __del__ = stop
+
+
+def main(args=sys.argv[1:]):
+    
+    # parse command line options
+    from optparse import OptionParser
+    parser = OptionParser()
+    parser.add_option('-p', '--port', dest='port', 
+                      type="int", default=8888,
+                      help="port to run the server on [DEFAULT: %default]")
+    parser.add_option('-H', '--host', dest='host',
+                      default='127.0.0.1',
+                      help="host [DEFAULT: %default]")
+    parser.add_option('-d', '--docroot', dest='docroot',
+                      default=os.getcwd(),
+                      help="directory to serve files from [DEFAULT: %default]")
+    parser.add_option('--test', dest='test',
+                      action='store_true', default=False,
+                      help='run the tests and exit')
+    options, args = parser.parse_args(args)
+    if args:
+        parser.print_help()
+        parser.exit()
+
+    # create the server
+    kwargs = options.__dict__.copy()
+    test = kwargs.pop('test')
+    server = MozHttpd(**kwargs)
+
+    if test:
+        server.start()
+        server.testServer()
+    else:
+        server.start(block=True)
+
+if __name__ == '__main__':
+    main()

testing/mozbase/mozhttpd/setup.py

@@ -0,0 +1,72 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozhttpd.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Joel Maher <jmaher@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os
+from setuptools import setup
+
+try:
+    here = os.path.dirname(os.path.abspath(__file__))
+    description = file(os.path.join(here, 'README.md')).read()
+except IOError:
+    description = None
+
+version = '0.1'
+
+deps = []
+
+setup(name='mozhttpd',
+      version=version,
+      description="basic python webserver, tested with talos",
+      long_description=description,
+      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+      keywords='mozilla',
+      author='Joel Maher',
+      author_email='tools@lists.mozilla.org',
+      url='https://github.com/mozilla/mozbase/tree/master/mozhttpd',
+      license='MPL',
+      py_modules=['mozhttpd'],
+      packages=[],
+      include_package_data=True,
+      zip_safe=False,
+      install_requires=deps,
+      entry_points="""
+      # -*- Entry points: -*-
+      [console_scripts]
+      mozhttpd = mozhttpd:main
+      """,
+      )
+

testing/mozbase/mozinfo/README.md

@@ -0,0 +1,62 @@
+Throughout [mozmill](https://developer.mozilla.org/en/Mozmill)
+and other Mozilla python code, checking the underlying
+platform is done in many different ways.  The various checks needed
+lead to a lot of copy+pasting, leaving the reader to wonder....is this
+specific check necessary for (e.g.) an operating system?  Because
+information is not consolidated, checks are not done consistently, nor
+is it defined what we are checking for.
+
+[MozInfo](https://github.com/mozilla/mozbase/tree/master/mozinfo)
+proposes to solve this problem.  MozInfo is a bridge interface,
+making the underlying (complex) plethora of OS and architecture
+combinations conform to a subset of values of relavence to 
+Mozilla software. The current implementation exposes relavent key,
+values: `os`, `version`, `bits`, and `processor`.  Additionally, the
+service pack in use is available on the windows platform.
+
+
+# API Usage
+
+MozInfo is a python package.  Downloading the software and running
+`python setup.py develop` will allow you to do `import mozinfo`
+from python.  
+[mozinfo.py](https://github.com/mozilla/mozbase/blob/master/mozinfo/mozinfo.py)
+is the only file contained is this package,
+so if you need a single-file solution, you can just download or call
+this file through the web.
+
+The top level attributes (`os`, `version`, `bits`, `processor`) are
+available as module globals:
+
+    if mozinfo.os == 'win': ...
+
+In addition, mozinfo exports a dictionary, `mozinfo.info`, that
+contain these values.  mozinfo also exports:
+
+- `choices`: a dictionary of possible values for os, bits, and
+  processor
+- `main`: the console_script entry point for mozinfo
+- `unknown`: a singleton denoting a value that cannot be determined
+
+`unknown` has the string representation `"UNKNOWN"`. unknown will evaluate
+as `False` in python:
+
+    if not mozinfo.os: ... # unknown!
+
+
+# Command Line Usage
+
+MozInfo comes with a command line, `mozinfo` which may be used to
+diagnose one's current system.
+
+Example output:
+
+    os: linux
+    version: Ubuntu 10.10
+    bits: 32
+    processor: x86
+
+Three of these fields, os, bits, and processor, have a finite set of
+choices.  You may display the value of these choices using 
+`mozinfo --os`, `mozinfo --bits`, and `mozinfo --processor`. 
+`mozinfo --help` documents command-line usage.

testing/mozbase/mozinfo/mozinfo.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozinfo.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Jeff Hammel <jhammel@mozilla.com>
+#  Clint Talbert <ctalbert@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""
+file for interface to transform introspected system information to a format
+pallatable to Mozilla
+
+Information:
+- os : what operating system ['win', 'mac', 'linux', ...]
+- bits : 32 or 64
+- processor : processor architecture ['x86', 'x86_64', 'ppc', ...]
+- version : operating system version string
+
+For windows, the service pack information is also included
+"""
+
+# TODO: it might be a good idea of adding a system name (e.g. 'Ubuntu' for
+# linux) to the information; I certainly wouldn't want anyone parsing this
+# information and having behaviour depend on it
+
+import os
+import platform
+import re
+import sys
+
+# keep a copy of the os module since updating globals overrides this
+_os = os
+
+class unknown(object):
+    """marker class for unknown information"""
+    def __nonzero__(self):
+        return False
+    def __str__(self):
+        return 'UNKNOWN'
+unknown = unknown() # singleton
+
+# get system information
+info = {'os': unknown,
+        'processor': unknown,
+        'version': unknown,
+        'bits': unknown }
+(system, node, release, version, machine, processor) = platform.uname()
+(bits, linkage) = platform.architecture()
+
+# get os information and related data
+if system in ["Microsoft", "Windows"]:
+    info['os'] = 'win'
+    # There is a Python bug on Windows to determine platform values
+    # http://bugs.python.org/issue7860
+    if "PROCESSOR_ARCHITEW6432" in os.environ:
+        processor = os.environ.get("PROCESSOR_ARCHITEW6432", processor)
+    else:
+        processor = os.environ.get('PROCESSOR_ARCHITECTURE', processor)
+        system = os.environ.get("OS", system).replace('_', ' ')
+        service_pack = os.sys.getwindowsversion()[4]
+        info['service_pack'] = service_pack
+elif system == "Linux":
+    (distro, version, codename) = platform.dist()
+    version = "%s %s" % (distro, version)
+    if not processor:
+        processor = machine
+    info['os'] = 'linux'
+elif system == "Darwin":
+    (release, versioninfo, machine) = platform.mac_ver()
+    version = "OS X %s" % release
+    info['os'] = 'mac'
+elif sys.platform in ('solaris', 'sunos5'):
+    info['os'] = 'unix'
+    version = sys.platform
+info['version'] = version # os version
+
+# processor type and bits
+if processor in ["i386", "i686"]:
+    if bits == "32bit":
+        processor = "x86"
+    elif bits == "64bit":
+        processor = "x86_64"
+elif processor == "AMD64":
+    bits = "64bit"
+    processor = "x86_64"
+elif processor == "Power Macintosh":
+    processor = "ppc"
+bits = re.search('(\d+)bit', bits).group(1)
+info.update({'processor': processor,
+             'bits': int(bits),
+            })
+
+# standard value of choices, for easy inspection
+choices = {'os': ['linux', 'win', 'mac', 'unix'],
+           'bits': [32, 64],
+           'processor': ['x86', 'x86_64', 'ppc']}
+
+
+def sanitize(info):
+    """Do some sanitization of input values, primarily
+    to handle universal Mac builds."""
+    if "processor" in info and info["processor"] == "universal-x86-x86_64":
+        # If we're running on OS X 10.6 or newer, assume 64-bit
+        if release[:4] >= "10.6": # Note this is a string comparison
+            info["processor"] = "x86_64"
+            info["bits"] = 64
+        else:
+            info["processor"] = "x86"
+            info["bits"] = 32
+
+# method for updating information
+def update(new_info):
+    """update the info"""
+    info.update(new_info)
+    sanitize(info)
+    globals().update(info)
+
+    # convenience data for os access
+    for os_name in choices['os']:
+        globals()['is' + os_name.title()] = info['os'] == os_name
+    # unix is special
+    if isLinux:
+        globals()['isUnix'] = True
+
+update({})
+
+# exports
+__all__ = info.keys()
+__all__ += ['is' + os_name.title() for os_name in choices['os']]
+__all__ += ['info', 'unknown', 'main', 'choices', 'update']
+
+
+def main(args=None):
+
+    # parse the command line
+    from optparse import OptionParser
+    parser = OptionParser(description=__doc__)
+    for key in choices:
+        parser.add_option('--%s' % key, dest=key,
+                          action='store_true', default=False,
+                          help="display choices for %s" % key)
+    options, args = parser.parse_args()
+
+    # args are JSON blobs to override info
+    if args:
+        try:
+            from json import loads
+        except ImportError:
+            try:
+                from simplejson import loads
+            except ImportError:
+                def loads(string):
+                    """*really* simple json; will not work with unicode"""
+                    return eval(string, {'true': True, 'false': False, 'null': None})
+        for arg in args:
+            if _os.path.exists(arg):
+                string = file(arg).read()
+            else:
+                string = arg
+            update(loads(string))
+
+    # print out choices if requested
+    flag = False
+    for key, value in options.__dict__.items():
+        if value is True:
+            print '%s choices: %s' % (key, ' '.join([str(choice)
+                                                     for choice in choices[key]]))
+            flag = True
+    if flag: return
+
+    # otherwise, print out all info
+    for key, value in info.items():
+        print '%s: %s' % (key, value)
+
+if __name__ == '__main__':
+    main()

testing/mozbase/mozinfo/setup.py

@@ -0,0 +1,78 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozinfo.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011.
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hammel <jhammel@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+
+import os
+from setuptools import setup
+
+version = '0.3.3'
+
+# get documentation from the README
+try:
+    here = os.path.dirname(os.path.abspath(__file__))
+    description = file(os.path.join(here, 'README.md')).read()
+except (OSError, IOError):
+    description = ''
+
+# dependencies
+deps = []
+try:
+    import json
+except ImportError:
+    deps = ['simplejson']
+
+setup(name='mozinfo',
+      version=version,
+      description="file for interface to transform introspected system information to a format pallatable to Mozilla",
+      long_description=description,
+      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+      keywords='mozilla',
+      author='Jeff Hammel',
+      author_email='jhammel@mozilla.com',
+      url='https://wiki.mozilla.org/Auto-tools',
+      license='MPL',
+      py_modules=['mozinfo'],
+      packages=[],
+      include_package_data=True,
+      zip_safe=False,
+      install_requires=deps,
+      entry_points="""
+      # -*- Entry points: -*-
+      [console_scripts]
+      mozinfo = mozinfo:main
+      """,
+      )

testing/mozbase/mozinstall/README.md

@@ -0,0 +1,35 @@
+[Mozinstall](https://github.com/mozilla/mozbase/tree/master/mozinstall)
+is a python package for installing Mozilla applications on various platforms.
+
+For example, depending on the platform, Firefox can be distributed as a 
+zip, tar.bz2, exe or dmg file or cloned from a repository. Mozinstall takes the 
+hassle out of extracting and/or running these files and for convenience returns
+the full path to the application's binary in the install directory. In the case 
+that mozinstall is invoked from the command line, the binary path will be 
+printed to stdout.
+
+# Usage
+
+For command line options run mozinstall --help
+
+Mozinstall's main function is the install method
+
+    import mozinstall
+    mozinstall.install('path_to_install_file', dest='path_to_install_folder')
+
+The dest parameter defaults to the directory in which the install file is located.
+The install method accepts a third parameter called apps which tells mozinstall which 
+binary to search for. By default it will search for 'firefox', 'thunderbird' and 'fennec'
+so unless you are installing a different application, this parameter is unnecessary.
+
+# Error Handling
+
+Mozinstall throws two different types of exceptions:
+
+- mozinstall.InvalidSource is thrown when the source is not a recognized file type (zip, exe, tar.bz2, tar.gz, dmg)
+- mozinstall.InstallError is thrown when the installation fails for any reason. A traceback is provided.
+
+# Dependencies
+
+Mozinstall depends on the [mozinfo](https://github.com/mozilla/mozbase/tree/master/mozinfo) 
+package which is also found in the mozbase repository.

testing/mozbase/mozinstall/mozinstall.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozinstall.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Clint Talbert <ctalbert@mozilla.com>
+#  Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+from optparse import OptionParser
+import mozinfo
+import subprocess
+import zipfile
+import tarfile
+import sys
+import os
+
+_default_apps = ["firefox",
+                 "thunderbird",
+                 "fennec"]
+
+def install(src, dest=None, apps=_default_apps):
+    """
+    Installs a zip, exe, tar.gz, tar.bz2 or dmg file
+    src - the path to the install file
+    dest - the path to install to [default is os.path.dirname(src)]
+    returns - the full path to the binary in the installed folder
+              or None if the binary cannot be found
+    """
+    src = os.path.realpath(src)
+    assert(os.path.isfile(src))
+    if not dest:
+        dest = os.path.dirname(src)
+
+    trbk = None
+    try:
+        install_dir = None
+        if zipfile.is_zipfile(src) or tarfile.is_tarfile(src):
+            install_dir = _extract(src, dest)[0]
+        elif mozinfo.isMac and src.lower().endswith(".dmg"):
+            install_dir = _install_dmg(src, dest)
+        elif mozinfo.isWin and os.access(src, os.X_OK):
+            install_dir = _install_exe(src, dest)
+        else:
+            raise InvalidSource(src + " is not a recognized file type " +
+                                      "(zip, exe, tar.gz, tar.bz2 or dmg)")
+    except InvalidSource, e:
+        raise
+    except Exception, e:
+        cls, exc, trbk = sys.exc_info()
+        install_error = InstallError("Failed to install %s" % src)
+        raise install_error.__class__, install_error, trbk
+    finally:
+        # trbk won't get GC'ed due to circular reference
+        # http://docs.python.org/library/sys.html#sys.exc_info
+        del trbk
+
+    if install_dir:
+        return get_binary(install_dir, apps=apps)
+
+def get_binary(path, apps=_default_apps):
+    """
+    Finds the binary in the specified path
+    path - the path within which to search for the binary
+    returns - the full path to the binary in the folder
+              or None if the binary cannot be found
+    """
+    if mozinfo.isWin:
+        apps = [app + ".exe" for app in apps]
+    for root, dirs, files in os.walk(path):
+        for filename in files:
+            # os.access evaluates to False for some reason, so not using it
+            if filename in apps:
+                return os.path.realpath(os.path.join(root, filename))
+
+def _extract(path, extdir=None, delete=False):
+    """
+    Takes in a tar or zip file and extracts it to extdir
+    If extdir is not specified, extracts to os.path.dirname(path)
+    If delete is set to True, deletes the bundle at path
+    Returns the list of top level files that were extracted
+    """
+    if zipfile.is_zipfile(path):
+        bundle = zipfile.ZipFile(path)
+        namelist = bundle.namelist()
+    elif tarfile.is_tarfile(path):
+        bundle = tarfile.open(path)
+        namelist = bundle.getnames()
+    else:
+        return
+    if extdir is None:
+        extdir = os.path.dirname(path)
+    elif not os.path.exists(extdir):
+        os.makedirs(extdir)
+    bundle.extractall(path=extdir)
+    bundle.close()
+    if delete:
+        os.remove(path)
+    # namelist returns paths with forward slashes even in windows
+    top_level_files = [os.path.join(extdir, name) for name in namelist
+                             if len(name.rstrip('/').split('/')) == 1]
+    # namelist doesn't include folders in windows, append these to the list
+    if mozinfo.isWin:
+        for name in namelist:
+            root = name[:name.find('/')]
+            if root not in top_level_files:
+                top_level_files.append(root)
+    return top_level_files
+
+def _install_dmg(src, dest):
+    proc = subprocess.Popen("hdiutil attach " + src,
+                            shell=True,
+                            stdout=subprocess.PIPE)
+    try:
+        for data in proc.communicate()[0].split():
+            if data.find("/Volumes/") != -1:
+                appDir = data
+                break
+        for appFile in os.listdir(appDir):
+            if appFile.endswith(".app"):
+                 appName = appFile
+                 break
+        subprocess.call("cp -r " + os.path.join(appDir, appName) + " " + dest,
+                        shell=True)
+    finally:
+        subprocess.call("hdiutil detach " + appDir + " -quiet",
+                        shell=True)
+    return os.path.join(dest, appName)
+
+def _install_exe(src, dest):
+    # possibly gets around UAC in vista (still need to run as administrator)
+    os.environ['__compat_layer'] = "RunAsInvoker"
+    cmd = [src, "/S", "/D=" + os.path.realpath(dest)]
+    subprocess.call(cmd)
+    return dest
+
+def cli(argv=sys.argv[1:]):
+    parser = OptionParser()
+    parser.add_option("-s", "--source",
+                      dest="src",
+                      help="Path to installation file. "
+                           "Accepts: zip, exe, tar.bz2, tar.gz, and dmg")
+    parser.add_option("-d", "--destination",
+                      dest="dest",
+                      default=None,
+                      help="[optional] Directory to install application into")
+    parser.add_option("--app", dest="app",
+                      action="append",
+                      default=_default_apps,
+                      help="[optional] Application being installed. "
+                           "Should be lowercase, e.g: "
+                           "firefox, fennec, thunderbird, etc.")
+
+    (options, args) = parser.parse_args(argv)
+    if not options.src or not os.path.exists(options.src):
+        print "Error: must specify valid source"
+        return 2
+
+    # Run it
+    if os.path.isdir(options.src):
+        binary = get_binary(options.src, apps=options.app)
+    else:
+        binary = install(options.src, dest=options.dest, apps=options.app)
+    print binary
+
+class InvalidSource(Exception):
+    """
+    Thrown when the specified source is not a recognized
+    file type (zip, exe, tar.gz, tar.bz2 or dmg)
+    """
+
+class InstallError(Exception):
+    """
+    Thrown when the installation fails. Includes traceback
+    if available.
+    """
+
+if __name__ == "__main__":
+    sys.exit(cli())

testing/mozbase/mozinstall/setup.py

@@ -0,0 +1,79 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozinstall.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Clint Talbert <ctalbert@mozilla.com>
+#  Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os
+from setuptools import setup
+
+try:
+    here = os.path.dirname(os.path.abspath(__file__))
+    description = file(os.path.join(here, 'README.md')).read()
+except IOError:
+    description = None
+
+version = '0.3'
+
+deps = ['mozinfo']
+
+setup(name='mozInstall',
+      version=version,
+      description="This is a utility package for installing Mozilla applications on various platforms.",
+      long_description=description,
+      classifiers=['Environment :: Console',
+                   'Intended Audience :: Developers',
+                   'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)',
+                   'Natural Language :: English',
+                   'Operating System :: OS Independent',
+                   'Programming Language :: Python',
+                   'Topic :: Software Development :: Libraries :: Python Modules',
+                  ], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+      keywords='mozilla',
+      author='mdas',
+      author_email='mdas@mozilla.com',
+      url='https://github.com/mozilla/mozbase',
+      license='MPL',
+      py_modules=['mozinstall'],
+      packages=[],
+      include_package_data=True,
+      zip_safe=False,
+      install_requires=deps,
+      entry_points="""
+      # -*- Entry points: -*-
+      [console_scripts]
+      mozinstall = mozinstall:cli
+      """,
+      )

testing/mozbase/mozlog/README.md

@@ -0,0 +1,18 @@
+[Mozlog](https://github.com/mozilla/mozbase/tree/master/mozlog)
+is a python package intended to simplify and standardize logs in the Mozilla universe. 
+It wraps around python's [logging](http://docs.python.org/library/logging.html) 
+module and adds some additional functionality.
+
+# Usage
+
+Import mozlog instead of [logging](http://docs.python.org/library/logging.html) 
+(all functionality in the logging module is also available from the mozlog module). 
+To get a logger, call mozlog.getLogger passing in a name and the path to a log file.
+If no log file is specified, the logger will log to stdout.
+
+    import mozlog
+    logger = mozlog.getLogger('LOG_NAME', 'log_file_path')
+    logger.setLevel(mozlog.DEBUG)
+    logger.info('foo')
+    logger.testPass('bar')
+    mozlog.shutdown()

testing/mozbase/mozlog/mozlog/__init__.py

@@ -0,0 +1,36 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/ #
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozlog.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+from logger import *

testing/mozbase/mozlog/mozlog/logger.py

@@ -0,0 +1,125 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/ #
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozlog.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+from logging import getLogger as getSysLogger
+from logging import *
+
+_default_level = INFO
+_LoggerClass = getLoggerClass()
+
+# Define mozlog specific log levels
+START      = _default_level + 1
+END        = _default_level + 2
+PASS       = _default_level + 3
+KNOWN_FAIL = _default_level + 4
+FAIL       = _default_level + 5
+# Define associated text of log levels
+addLevelName(START, 'TEST-START')
+addLevelName(END, 'TEST-END')
+addLevelName(PASS, 'TEST-PASS')
+addLevelName(KNOWN_FAIL, 'TEST-KNOWN-FAIL')
+addLevelName(FAIL, 'TEST-UNEXPECTED-FAIL')
+
+class _MozLogger(_LoggerClass):
+    """
+    MozLogger class which adds three convenience log levels
+    related to automated testing in Mozilla
+    """
+    def testStart(self, message, *args, **kwargs):
+        self.log(START, message, *args, **kwargs)
+
+    def testEnd(self, message, *args, **kwargs):
+        self.log(END, message, *args, **kwargs)
+
+    def testPass(self, message, *args, **kwargs):
+        self.log(PASS, message, *args, **kwargs)
+
+    def testFail(self, message, *args, **kwargs):
+        self.log(FAIL, message, *args, **kwargs)
+
+    def testKnownFail(self, message, *args, **kwargs):
+        self.log(KNOWN_FAIL, message, *args, **kwargs)
+
+class _MozFormatter(Formatter):
+    """
+    MozFormatter class used for default formatting
+    This can easily be overriden with the log handler's setFormatter()
+    """
+    level_length = 0
+    max_level_length = len('TEST-START')
+
+    def __init__(self):
+        pass
+
+    def format(self, record):
+        record.message = record.getMessage()
+
+        # Handles padding so record levels align nicely
+        if len(record.levelname) > self.level_length:
+            pad = 0
+            if len(record.levelname) <= self.max_level_length:
+                self.level_length = len(record.levelname)
+        else:
+            pad = self.level_length - len(record.levelname) + 1
+        sep = '|'.rjust(pad)
+        fmt = '%(name)s %(levelname)s ' + sep + ' %(message)s'
+        return fmt % record.__dict__
+
+def getLogger(name, logfile=None):
+    """
+    Returns the logger with the specified name.
+    If the logger doesn't exist, it is created.
+
+    name       - The name of the logger to retrieve
+    [filePath] - If specified, the logger will log to the specified filePath
+                 Otherwise, the logger logs to stdout
+                 This parameter only has an effect if the logger doesn't exist
+    """
+    setLoggerClass(_MozLogger)
+
+    if name in Logger.manager.loggerDict:
+        return getSysLogger(name)
+
+    logger = getSysLogger(name)
+    logger.setLevel(_default_level)
+
+    if logfile:
+        handler = FileHandler(logfile)
+    else:
+        handler = StreamHandler()
+    handler.setFormatter(_MozFormatter())
+    logger.addHandler(handler)
+    return logger
+

testing/mozbase/mozlog/setup.py

@@ -0,0 +1,70 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/ #
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozlog.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os
+import sys
+from setuptools import setup, find_packages
+
+PACKAGE_NAME = "mozlog"
+PACKAGE_VERSION = "1.0"
+
+desc = """Robust log handling specialized for logging in the Mozilla universe"""
+# take description from README
+here = os.path.dirname(os.path.abspath(__file__))
+try:
+    description = file(os.path.join(here, 'README.md')).read()
+except IOError, OSError:
+    description = ''
+
+setup(name=PACKAGE_NAME,
+      version=PACKAGE_VERSION,
+      description=desc,
+      long_description=description,
+      author='Andrew Halberstadt, Mozilla',
+      author_email='halbersa@gmail.com',
+      url='http://github.com/ahal/mozbase',
+      license='MPL 1.1/GPL 2.0/LGPL 2.1',
+      packages=find_packages(exclude=['legacy']),
+      zip_safe=False,
+      platforms =['Any'],
+      classifiers=['Development Status :: 4 - Beta',
+                   'Environment :: Console',
+                   'Intended Audience :: Developers',
+                   'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)',
+                   'Operating System :: OS Independent',
+                   'Topic :: Software Development :: Libraries :: Python Modules',
+                  ]
+     )

testing/mozbase/mozprocess/README.md

@@ -0,0 +1,34 @@
+[mozprocess](https://github.com/mozilla/mozbase/tree/master/mozprocess)
+provides python process management via an operating system 
+and platform transparent interface to Mozilla platforms of interest.
+Mozprocess aims to provide the ability
+to robustly terminate a process (by timeout or otherwise), along with
+any child processes, on Windows, OS X, and Linux. Mozprocess utilizes
+and extends `subprocess.Popen` to these ends.
+
+
+# API
+
+[mozprocess.processhandler:ProcessHandler](https://github.com/mozilla/mozbase/blob/master/mozprocess/mozprocess/processhandler.py)
+is the central exposed API for mozprocess.  `ProcessHandler` utilizes
+a contained subclass of [subprocess.Popen](http://docs.python.org/library/subprocess.html),
+`Process`, which does the brunt of the process management.
+
+Basic usage:
+
+    process = ProcessHandler(['command', '-line', 'arguments'],
+                             cwd=None, # working directory for cmd; defaults to None
+                             env={},   # environment to use for the process; defaults to os.environ
+                             )         
+    exit_code = process.waitForFinish(timeout=60) # seconds
+
+See an example in https://github.com/mozilla/mozbase/blob/master/mutt/mutt/tests/python/testprofilepath.py
+
+`ProcessHandler` may be subclassed to handle process timeouts (by overriding
+the `onTimeout()` method), process completion (by overriding 
+`onFinish()`), and to process the command output (by overriding 
+`processOutputLine()`).
+
+# TODO
+
+- Document improvements over `subprocess.Popen.kill`

testing/mozbase/mozprocess/mozprocess/__init__.py

@@ -0,0 +1,40 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprocess.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Clint Talbert <ctalbert@mozilla.com>
+#  Jonathan Griffin <jgriffin@mozilla.com>
+#  Jeff Hammel <jhammel@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+from processhandler import *

testing/mozbase/mozprocess/mozprocess/pid.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprocess.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Clint Talbert <ctalbert@mozilla.com>
+#  Jonathan Griffin <jgriffin@mozilla.com>
+#  Jeff Hammel <jhammel@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os
+import mozinfo
+import shlex
+import subprocess
+import sys
+
+# determine the platform-specific invocation of `ps`
+if mozinfo.isMac:
+    psarg = '-Acj'
+elif mozinfo.isLinux:
+    psarg = 'axwww'
+else:
+    psarg = 'ax'
+
+def ps(arg=psarg):
+    """
+    python front-end to `ps`
+    http://en.wikipedia.org/wiki/Ps_%28Unix%29
+    returns a list of process dicts based on the `ps` header
+    """
+    retval = []
+    process = subprocess.Popen(['ps', arg], stdout=subprocess.PIPE)
+    stdout, _ = process.communicate()
+    header = None
+    for line in stdout.splitlines():
+        line = line.strip()
+        if header is None:
+            # first line is the header
+            header = line.split()
+            continue
+        split = line.split(None, len(header)-1)
+        process_dict = dict(zip(header, split))
+        retval.append(process_dict)
+    return retval
+
+def running_processes(name, psarg=psarg, defunct=True):
+    """
+    returns a list of
+    {'PID': PID of process (int)
+     'command': command line of process (list)}
+     with the executable named `name`.
+     - defunct: whether to return defunct processes
+    """
+    retval = []
+    for process in ps(psarg):
+        command = process['COMMAND']
+        command = shlex.split(command)
+        if command[-1] == '<defunct>':
+            command = command[:-1]
+            if not command or not defunct:
+                continue
+        if 'STAT' in process and not defunct:
+            if process['STAT'] == 'Z+':
+                continue
+        prog = command[0]
+        basename = os.path.basename(prog)
+        if basename == name:
+            retval.append((int(process['PID']), command))
+    return retval
+
+def get_pids(name):
+    """Get all the pids matching name"""
+
+    if mozinfo.isWin:
+        # use the windows-specific implementation
+        import wpk
+        return wpk.get_pids(name)
+    else:
+        return [pid for pid,_ in running_processes(name)]
+
+if __name__ == '__main__':
+    pids = set()
+    for i in sys.argv[1:]:
+        pids.update(get_pids(i))
+    for i in sorted(pids):
+        print i

testing/mozbase/mozprocess/mozprocess/processhandler.py

@@ -0,0 +1,758 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprocess.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Clint Talbert <ctalbert@mozilla.com>
+#  Jonathan Griffin <jgriffin@mozilla.com>
+#  Jeff Hammel <jhammel@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import logging
+import mozinfo
+import os
+import select
+import signal
+import subprocess
+import sys
+import threading
+import time
+from Queue import Queue
+from datetime import datetime, timedelta
+
+__all__ = ['ProcessHandlerMixin', 'ProcessHandler']
+
+if mozinfo.isWin:
+    import ctypes, ctypes.wintypes, msvcrt
+    from ctypes import sizeof, addressof, c_ulong, byref, POINTER, WinError
+    import winprocess
+    from qijo import JobObjectAssociateCompletionPortInformation, JOBOBJECT_ASSOCIATE_COMPLETION_PORT
+
+class ProcessHandlerMixin(object):
+    """Class which represents a process to be executed."""
+
+    class Process(subprocess.Popen):
+        """
+        Represents our view of a subprocess.
+        It adds a kill() method which allows it to be stopped explicitly.
+        """
+
+        MAX_IOCOMPLETION_PORT_NOTIFICATION_DELAY = 180
+        MAX_PROCESS_KILL_DELAY = 30
+
+        def __init__(self,
+                     args,
+                     bufsize=0,
+                     executable=None,
+                     stdin=None,
+                     stdout=None,
+                     stderr=None,
+                     preexec_fn=None,
+                     close_fds=False,
+                     shell=False,
+                     cwd=None,
+                     env=None,
+                     universal_newlines=False,
+                     startupinfo=None,
+                     creationflags=0,
+                     ignore_children=False):
+
+            # Parameter for whether or not we should attempt to track child processes
+            self._ignore_children = ignore_children
+
+            if not self._ignore_children and not mozinfo.isWin:
+                # Set the process group id for linux systems
+                # Sets process group id to the pid of the parent process
+                # NOTE: This prevents you from using preexec_fn and managing
+                #       child processes, TODO: Ideally, find a way around this
+                def setpgidfn():
+                    os.setpgid(0, 0)
+                preexec_fn = setpgidfn
+
+            try:
+                subprocess.Popen.__init__(self, args, bufsize, executable,
+                                          stdin, stdout, stderr,
+                                          preexec_fn, close_fds,
+                                          shell, cwd, env,
+                                          universal_newlines, startupinfo, creationflags)
+            except OSError, e:
+                print >> sys.stderr, args
+                raise
+
+        def __del__(self, _maxint=sys.maxint):
+            if mozinfo.isWin:
+                if self._handle:
+                    self._internal_poll(_deadstate=_maxint)
+                if self._handle or self._job or self._io_port:
+                    self._cleanup()
+            else:
+                subprocess.Popen.__del__(self)
+
+        def kill(self):
+            self.returncode = 0
+            if mozinfo.isWin:
+                if not self._ignore_children and self._handle and self._job:
+                    winprocess.TerminateJobObject(self._job, winprocess.ERROR_CONTROL_C_EXIT)
+                    self.returncode = winprocess.GetExitCodeProcess(self._handle)
+                elif self._handle:
+                    try:
+                        winprocess.TerminateProcess(self._handle, winprocess.ERROR_CONTROL_C_EXIT)
+                    except:
+                        raise OSError("Could not terminate process")
+                    finally:
+                        self.returncode = winprocess.GetExitCodeProcess(self._handle)
+                        self._cleanup()
+                else:
+                    pass
+            else:
+                if not self._ignore_children:
+                    try:
+                        os.killpg(self.pid, signal.SIGKILL)
+                    except BaseException, e:
+                        if getattr(e, "errno", None) != 3:
+                            # Error 3 is "no such process", which is ok
+                            print >> sys.stderr, "Could not kill process, could not find pid: %s" % self.pid
+                    finally:
+                        # Try to get the exit status
+                        if self.returncode is None:
+                            self.returncode = subprocess.Popen._internal_poll(self)
+
+                else:
+                    os.kill(self.pid, signal.SIGKILL)
+                    if self.returncode is None:
+                        self.returncode = subprocess.Popen._internal_poll(self)
+
+            self._cleanup()
+            return self.returncode
+
+        def wait(self):
+            """ Popen.wait
+                Called to wait for a running process to shut down and return
+                its exit code
+                Returns the main process's exit code
+            """
+            # This call will be different for each OS
+            self.returncode = self._wait()
+            self._cleanup()
+            return self.returncode
+
+        """ Private Members of Process class """
+
+        if mozinfo.isWin:
+            # Redefine the execute child so that we can track process groups
+            def _execute_child(self, args, executable, preexec_fn, close_fds,
+                               cwd, env, universal_newlines, startupinfo,
+                               creationflags, shell,
+                               p2cread, p2cwrite,
+                               c2pread, c2pwrite,
+                               errread, errwrite):
+                if not isinstance(args, basestring):
+                    args = subprocess.list2cmdline(args)
+
+                # Always or in the create new process group
+                creationflags |= winprocess.CREATE_NEW_PROCESS_GROUP
+
+                if startupinfo is None:
+                    startupinfo = winprocess.STARTUPINFO()
+
+                if None not in (p2cread, c2pwrite, errwrite):
+                    startupinfo.dwFlags |= winprocess.STARTF_USESTDHANDLES
+                    startupinfo.hStdInput = int(p2cread)
+                    startupinfo.hStdOutput = int(c2pwrite)
+                    startupinfo.hStdError = int(errwrite)
+                if shell:
+                    startupinfo.dwFlags |= winprocess.STARTF_USESHOWWINDOW
+                    startupinfo.wShowWindow = winprocess.SW_HIDE
+                    comspec = os.environ.get("COMSPEC", "cmd.exe")
+                    args = comspec + " /c " + args
+
+                # determine if we can create create a job
+                canCreateJob = winprocess.CanCreateJobObject()
+
+                # Ensure we write a warning message if we are falling back
+                if not canCreateJob and not self._ignore_children:
+                    # We can't create job objects AND the user wanted us to
+                    # Warn the user about this.
+                    print >> sys.stderr, "ProcessManager UNABLE to use job objects to manage child processes"
+
+                # set process creation flags
+                creationflags |= winprocess.CREATE_SUSPENDED
+                creationflags |= winprocess.CREATE_UNICODE_ENVIRONMENT
+                if canCreateJob:
+                    creationflags |= winprocess.CREATE_BREAKAWAY_FROM_JOB
+                else:
+                    # Since we've warned, we just log info here to inform you
+                    # of the consequence of setting ignore_children = True
+                    print "ProcessManager NOT managing child processes"
+
+                # create the process
+                hp, ht, pid, tid = winprocess.CreateProcess(
+                    executable, args,
+                    None, None, # No special security
+                    1, # Must inherit handles!
+                    creationflags,
+                    winprocess.EnvironmentBlock(env),
+                    cwd, startupinfo)
+                self._child_created = True
+                self._handle = hp
+                self._thread = ht
+                self.pid = pid
+                self.tid = tid
+
+                if canCreateJob:
+                    try:
+                        # We create a new job for this process, so that we can kill
+                        # the process and any sub-processes
+                        # Create the IO Completion Port
+                        self._io_port = winprocess.CreateIoCompletionPort()
+                        self._job = winprocess.CreateJobObject()
+
+                        # Now associate the io comp port and the job object
+                        joacp = JOBOBJECT_ASSOCIATE_COMPLETION_PORT(winprocess.COMPKEY_JOBOBJECT,
+                                                                    self._io_port)
+                        winprocess.SetInformationJobObject(self._job,
+                                                          JobObjectAssociateCompletionPortInformation,
+                                                          addressof(joacp),
+                                                          sizeof(joacp)
+                                                          )
+
+                        # Assign the job object to the process
+                        winprocess.AssignProcessToJobObject(self._job, int(hp))
+
+                        # It's overkill, but we use Queue to signal between threads
+                        # because it handles errors more gracefully than event or condition.
+                        self._process_events = Queue()
+
+                        # Spin up our thread for managing the IO Completion Port
+                        self._procmgrthread = threading.Thread(target = self._procmgr)
+                    except:
+                        print >> sys.stderr, """Exception trying to use job objects;
+falling back to not using job objects for managing child processes"""
+                        # Ensure no dangling handles left behind
+                        self._cleanup_job_io_port()
+                else:
+                    self._job = None
+
+                winprocess.ResumeThread(int(ht))
+                if self._procmgrthread:
+                    self._procmgrthread.start()
+                ht.Close()
+
+                for i in (p2cread, c2pwrite, errwrite):
+                    if i is not None:
+                        i.Close()
+
+            # Windows Process Manager - watches the IO Completion Port and
+            # keeps track of child processes
+            def _procmgr(self):
+                if not (self._io_port) or not (self._job):
+                    return
+
+                try:
+                    self._poll_iocompletion_port()
+                except KeyboardInterrupt:
+                    raise KeyboardInterrupt
+
+            def _poll_iocompletion_port(self):
+                # Watch the IO Completion port for status
+                self._spawned_procs = {}
+                countdowntokill = 0
+
+                while True:
+                    msgid = c_ulong(0)
+                    compkey = c_ulong(0)
+                    pid = c_ulong(0)
+                    portstatus = winprocess.GetQueuedCompletionStatus(self._io_port,
+                                                                      byref(msgid),
+                                                                      byref(compkey),
+                                                                      byref(pid),
+                                                                      5000)
+
+                    # If the countdowntokill has been activated, we need to check
+                    # if we should start killing the children or not.
+                    if countdowntokill != 0:
+                        diff = datetime.now() - countdowntokill
+                        # Arbitrarily wait 3 minutes for windows to get its act together
+                        # Windows sometimes takes a small nap between notifying the
+                        # IO Completion port and actually killing the children, and we
+                        # don't want to mistake that situation for the situation of an unexpected
+                        # parent abort (which is what we're looking for here).
+                        if diff.seconds > self.MAX_IOCOMPLETION_PORT_NOTIFICATION_DELAY:
+                            print >> sys.stderr, "Parent process exited without \
+                                                  killing children, attempting to kill children"
+                            self.kill()
+                            self._process_events.put({self.pid: 'FINISHED'})
+
+                    if not portstatus:
+                        # Check to see what happened
+                        errcode = winprocess.GetLastError()
+                        if errcode == winprocess.ERROR_ABANDONED_WAIT_0:
+                            # Then something has killed the port, break the loop
+                            print >> sys.stderr, "IO Completion Port unexpectedly closed"
+                            break
+                        elif errcode == winprocess.WAIT_TIMEOUT:
+                            # Timeouts are expected, just keep on polling
+                            continue
+                        else:
+                            print >> sys.stderr, "Error Code %s trying to query IO Completion Port, exiting" % errcode
+                            raise WinError(errcode)
+                            break
+
+                    if compkey.value == winprocess.COMPKEY_TERMINATE.value:
+                        # Then we're done
+                        break
+
+                    # Check the status of the IO Port and do things based on it
+                    if compkey.value == winprocess.COMPKEY_JOBOBJECT.value:
+                        if msgid.value == winprocess.JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO:
+                            # No processes left, time to shut down
+                            # Signal anyone waiting on us that it is safe to shut down
+                            self._process_events.put({self.pid: 'FINISHED'})
+                            break
+                        elif msgid.value == winprocess.JOB_OBJECT_MSG_NEW_PROCESS:
+                            # New Process started
+                            # Add the child proc to our list in case our parent flakes out on us
+                            # without killing everything.
+                            if pid.value != self.pid:
+                                self._spawned_procs[pid.value] = 1
+                        elif msgid.value == winprocess.JOB_OBJECT_MSG_EXIT_PROCESS:
+                            # One process exited normally
+                            if pid.value == self.pid and len(self._spawned_procs) > 0:
+                                # Parent process dying, start countdown timer
+                                countdowntokill = datetime.now()
+                            elif pid.value in self._spawned_procs:
+                                # Child Process died remove from list
+                                del(self._spawned_procs[pid.value])
+                        elif msgid.value == winprocess.JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS:
+                            # One process existed abnormally
+                            if pid.value == self.pid and len(self._spawned_procs) > 0:
+                                # Parent process dying, start countdown timer
+                                countdowntokill = datetime.now()
+                            elif pid.value in self._spawned_procs:
+                                # Child Process died remove from list
+                                del self._spawned_procs[pid.value]
+                        else:
+                            # We don't care about anything else
+                            pass
+
+            def _wait(self):
+
+                # First, check to see if the process is still running
+                if self._handle:
+                    self.returncode = winprocess.GetExitCodeProcess(self._handle)
+                else:
+                    # Dude, the process is like totally dead!
+                    return self.returncode
+
+                if self._job and self._procmgrthread.is_alive():
+                    # Then we are managing with IO Completion Ports
+                    # wait on a signal so we know when we have seen the last
+                    # process come through.
+                    # We use queues to synchronize between the thread and this
+                    # function because events just didn't have robust enough error
+                    # handling on pre-2.7 versions
+                    try:
+                        # timeout is the max amount of time the procmgr thread will wait for
+                        # child processes to shutdown before killing them with extreme prejudice.
+                        item = self._process_events.get(timeout=self.MAX_IOCOMPLETION_PORT_NOTIFICATION_DELAY +
+                                                                self.MAX_PROCESS_KILL_DELAY)
+                        if item[self.pid] == 'FINISHED':
+                            self._process_events.task_done()
+                    except:
+                        raise OSError("IO Completion Port failed to signal process shutdown")
+                    finally:
+                        # Either way, let's try to get this code
+                        self.returncode = winprocess.GetExitCodeProcess(self._handle)
+                        self._cleanup()
+
+                else:
+                    # Not managing with job objects, so all we can reasonably do
+                    # is call waitforsingleobject and hope for the best
+
+                    # First, make sure we have not already ended
+                    if self.returncode != winprocess.STILL_ACTIVE:
+                        self._cleanup()
+                        return self.returncode
+
+                    rc = None
+                    if self._handle:
+                        rc = winprocess.WaitForSingleObject(self._handle, -1)
+
+                    if rc == winprocess.WAIT_TIMEOUT:
+                        # The process isn't dead, so kill it
+                        print "Timed out waiting for process to close, attempting TerminateProcess"
+                        self.kill()
+                    elif rc == winprocess.WAIT_OBJECT_0:
+                        # We caught WAIT_OBJECT_0, which indicates all is well
+                        print "Single process terminated successfully"
+                        self.returncode = winprocess.GetExitCodeProcess(self._handle)
+                    else:
+                        # An error occured we should probably throw
+                        rc = winprocess.GetLastError()
+                        if rc:
+                            raise WinError(rc)
+
+                    self._cleanup()
+                    return self.returncode
+
+            def _cleanup_job_io_port(self):
+                """ Do the job and IO port cleanup separately because there are
+                    cases where we want to clean these without killing _handle
+                    (i.e. if we fail to create the job object in the first place)
+                """
+                if self._job and self._job != winprocess.INVALID_HANDLE_VALUE:
+                    self._job.Close()
+                    self._job = None
+                else:
+                    # If windows already freed our handle just set it to none
+                    # (saw this intermittently while testing)
+                    self._job = None
+
+                if self._io_port and self._io_port != winprocess.INVALID_HANDLE_VALUE:
+                    self._io_port.Close()
+                    self._io_port = None
+                else:
+                    self._io_port = None
+
+                if self._procmgrthread:
+                    self._procmgrthread = None
+
+            def _cleanup(self):
+                self._cleanup_job_io_port()
+                if self._thread and self._thread != winprocess.INVALID_HANDLE_VALUE:
+                    self._thread.Close()
+                    self._thread = None
+                else:
+                    self._thread = None
+
+                if self._handle and self._handle != winprocess.INVALID_HANDLE_VALUE:
+                    self._handle.Close()
+                    self._handle = None
+                else:
+                    self._handle = None
+
+        elif mozinfo.isMac or mozinfo.isUnix:
+
+            def _wait(self):
+                """ Haven't found any reason to differentiate between these platforms
+                    so they all use the same wait callback.  If it is necessary to
+                    craft different styles of wait, then a new _wait method
+                    could be easily implemented.
+                """
+
+                if not self._ignore_children:
+                    try:
+                        # os.waitpid returns a (pid, status) tuple
+                        return os.waitpid(self.pid, 0)[1]
+                    except OSError, e:
+                        if getattr(e, "errno", None) != 10:
+                            # Error 10 is "no child process", which could indicate normal
+                            # close
+                            print >> sys.stderr, "Encountered error waiting for pid to close: %s" % e
+                            raise
+                        return 0
+
+                else:
+                    # For non-group wait, call base class
+                    subprocess.Popen.wait(self)
+                    return self.returncode
+
+            def _cleanup(self):
+                pass
+
+        else:
+            # An unrecognized platform, we will call the base class for everything
+            print >> sys.stderr, "Unrecognized platform, process groups may not be managed properly"
+
+            def _wait(self):
+                self.returncode = subprocess.Popen.wait(self)
+                return self.returncode
+
+            def _cleanup(self):
+                pass
+
+    def __init__(self,
+                 cmd,
+                 args=None,
+                 cwd=None,
+                 env=os.environ.copy(),
+                 ignore_children = False,
+                 processOutputLine=(),
+                 onTimeout=(),
+                 onFinish=(),
+                 **kwargs):
+        """
+        cmd = Command to run
+        args = array of arguments (defaults to None)
+        cwd = working directory for cmd (defaults to None)
+        env = environment to use for the process (defaults to os.environ)
+        ignore_children = when True, causes system to ignore child processes,
+        defaults to False (which tracks child processes)
+        processOutputLine = handlers to process the output line
+        onTimeout = handlers for timeout event
+        kwargs = keyword args to pass directly into Popen
+
+        NOTE: Child processes will be tracked by default.  If for any reason
+        we are unable to track child processes and ignore_children is set to False,
+        then we will fall back to only tracking the root process.  The fallback
+        will be logged.
+        """
+        self.cmd = cmd
+        self.args = args
+        self.cwd = cwd
+        self.env = env
+        self.didTimeout = False
+        self._ignore_children = ignore_children
+        self.keywordargs = kwargs
+
+        # handlers
+        self.processOutputLineHandlers = list(processOutputLine)
+        self.onTimeoutHandlers = list(onTimeout)
+        self.onFinishHandlers = list(onFinish)
+
+        # It is common for people to pass in the entire array with the cmd and
+        # the args together since this is how Popen uses it.  Allow for that.
+        if not isinstance(self.cmd, list):
+            self.cmd = [self.cmd]
+
+        if self.args:
+            self.cmd = self.cmd + self.args
+
+    @property
+    def timedOut(self):
+        """True if the process has timed out."""
+        return self.didTimeout
+
+    @property
+    def commandline(self):
+        """the string value of the command line"""
+        return subprocess.list2cmdline([self.cmd] + self.args)
+
+    def run(self):
+        """Starts the process.  waitForFinish must be called to allow the
+           process to complete.
+        """
+        self.didTimeout = False
+        self.startTime = datetime.now()
+        self.proc = self.Process(self.cmd,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.STDOUT,
+                                 cwd=self.cwd,
+                                 env=self.env,
+                                 ignore_children = self._ignore_children,
+                                 **self.keywordargs)
+
+    def kill(self):
+        """
+          Kills the managed process and if you created the process with
+          'ignore_children=False' (the default) then it will also
+          also kill all child processes spawned by it.
+          If you specified 'ignore_children=True' when creating the process,
+          only the root process will be killed.
+
+          Note that this does not manage any state, save any output etc,
+          it immediately kills the process.
+        """
+        return self.proc.kill()
+
+    def readWithTimeout(self, f, timeout):
+        """
+          Try to read a line of output from the file object |f|.
+          |f| must be a  pipe, like the |stdout| member of a subprocess.Popen
+          object created with stdout=PIPE. If no output
+          is received within |timeout| seconds, return a blank line.
+          Returns a tuple (line, did_timeout), where |did_timeout| is True
+          if the read timed out, and False otherwise.
+
+          Calls a private member because this is a different function based on
+          the OS
+        """
+        return self._readWithTimeout(f, timeout)
+
+    def processOutputLine(self, line):
+        """Called for each line of output that a process sends to stdout/stderr.
+        """
+        for handler in self.processOutputLineHandlers:
+            handler(line)
+
+    def onTimeout(self):
+        """Called when a process times out."""
+        for handler in self.onTimeoutHandlers:
+            handler()
+
+    def onFinish(self):
+        """Called when a process finishes without a timeout."""
+        for handler in self.onFinishHandlers:
+            handler()
+
+    def waitForFinish(self, timeout=None, outputTimeout=None):
+        """
+        Handle process output until the process terminates or times out.
+
+        If timeout is not None, the process will be allowed to continue for
+        that number of seconds before being killed.
+
+        If outputTimeout is not None, the process will be allowed to continue
+        for that number of seconds without producing any output before
+        being killed.
+        """
+
+        if not hasattr(self, 'proc'):
+            self.run()
+
+        self.didTimeout = False
+        logsource = self.proc.stdout
+
+        lineReadTimeout = None
+        if timeout:
+            lineReadTimeout = timeout - (datetime.now() - self.startTime).seconds
+        elif outputTimeout:
+            lineReadTimeout = outputTimeout
+
+        (line, self.didTimeout) = self.readWithTimeout(logsource, lineReadTimeout)
+        while line != "" and not self.didTimeout:
+            self.processOutputLine(line.rstrip())
+            if timeout:
+                lineReadTimeout = timeout - (datetime.now() - self.startTime).seconds
+            (line, self.didTimeout) = self.readWithTimeout(logsource, lineReadTimeout)
+
+
+        if self.didTimeout:
+            self.proc.kill()
+            self.onTimeout()
+        else:
+            self.onFinish()
+
+        status = self.proc.wait()
+        return status
+
+
+    ### Private methods from here on down. Thar be dragons.
+
+    if mozinfo.isWin:
+        # Windows Specific private functions are defined in this block
+        PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
+        GetLastError = ctypes.windll.kernel32.GetLastError
+
+        def _readWithTimeout(self, f, timeout):
+            if timeout is None:
+                # shortcut to allow callers to pass in "None" for no timeout.
+                return (f.readline(), False)
+            x = msvcrt.get_osfhandle(f.fileno())
+            l = ctypes.c_long()
+            done = time.time() + timeout
+            while time.time() < done:
+                if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0:
+                    err = self.GetLastError()
+                    if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE
+                        return ('', False)
+                    else:
+                        raise OSError("readWithTimeout got error: %d", err)
+                if l.value > 0:
+                    # we're assuming that the output is line-buffered,
+                    # which is not unreasonable
+                    return (f.readline(), False)
+                time.sleep(0.01)
+            return ('', True)
+
+    else:
+        # Generic
+        def _readWithTimeout(self, f, timeout):
+            try:
+                (r, w, e) = select.select([f], [], [], timeout)
+            except:
+                # TODO: return a blank line?
+                return ('', True)
+
+            if len(r) == 0:
+                return ('', True)
+            return (f.readline(), False)
+
+
+### default output handlers
+### these should be callables that take the output line
+
+def print_output(line):
+    print line
+
+class StoreOutput(object):
+    """accumulate stdout"""
+
+    def __init__(self):
+        self.output = []
+
+    def __call__(self, line):
+        self.output.append(line)
+
+class LogOutput(object):
+    """pass output to a file"""
+
+    def __init__(self, filename):
+        self.filename = filename
+        self.file = None
+
+    def __call__(self, line):
+        if self.file is None:
+            self.file = file(self.filename, 'a')
+        self.file.write(line + '\n')
+        self.file.flush()
+
+    def __del__(self):
+        if self.file is not None:
+            self.file.close()
+
+### front end class with the default handlers
+
+class ProcessHandler(ProcessHandlerMixin):
+
+    def __init__(self, cmd, logfile=None, storeOutput=True, **kwargs):
+        """
+        If storeOutput=True, the output produced by the process will be saved
+        as self.output.
+
+        If logfile is not None, the output produced by the process will be
+        appended to the given file.
+        """
+
+        kwargs.setdefault('processOutputLine', []).append(print_output)
+
+        if logfile:
+            logoutput = LogOutput(logfile)
+            kwargs['processOutputLine'].append(logoutput)
+
+        self.output = None
+        if storeOutput:
+            storeoutput = StoreOutput()
+            self.output = storeoutput.output
+            kwargs['processOutputLine'].append(storeoutput)
+
+        ProcessHandlerMixin.__init__(self, cmd, **kwargs)

testing/mozbase/mozprocess/mozprocess/qijo.py

@@ -0,0 +1,175 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprocess.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Clint Talbert <ctalbert@mozilla.com>
+#  Jonathan Griffin <jgriffin@mozilla.com>
+#  Jeff Hammel <jhammel@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+from ctypes import c_void_p, POINTER, sizeof, Structure, windll, WinError, WINFUNCTYPE, addressof, c_size_t, c_ulong
+from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LARGE_INTEGER
+
+LPVOID = c_void_p
+LPDWORD = POINTER(DWORD)
+SIZE_T = c_size_t
+ULONG_PTR = POINTER(c_ulong)
+
+# A ULONGLONG is a 64-bit unsigned integer.
+# Thus there are 8 bytes in a ULONGLONG.
+# XXX why not import c_ulonglong ?
+ULONGLONG = BYTE * 8
+
+class IO_COUNTERS(Structure):
+    # The IO_COUNTERS struct is 6 ULONGLONGs.
+    # TODO: Replace with non-dummy fields.
+    _fields_ = [('dummy', ULONGLONG * 6)]
+
+class JOBOBJECT_BASIC_ACCOUNTING_INFORMATION(Structure):
+    _fields_ = [('TotalUserTime', LARGE_INTEGER),
+                ('TotalKernelTime', LARGE_INTEGER),
+                ('ThisPeriodTotalUserTime', LARGE_INTEGER),
+                ('ThisPeriodTotalKernelTime', LARGE_INTEGER),
+                ('TotalPageFaultCount', DWORD),
+                ('TotalProcesses', DWORD),
+                ('ActiveProcesses', DWORD),
+                ('TotalTerminatedProcesses', DWORD)]
+
+class JOBOBJECT_BASIC_AND_IO_ACCOUNTING_INFORMATION(Structure):
+    _fields_ = [('BasicInfo', JOBOBJECT_BASIC_ACCOUNTING_INFORMATION),
+                ('IoInfo', IO_COUNTERS)]
+
+# see http://msdn.microsoft.com/en-us/library/ms684147%28VS.85%29.aspx
+class JOBOBJECT_BASIC_LIMIT_INFORMATION(Structure):
+    _fields_ = [('PerProcessUserTimeLimit', LARGE_INTEGER),
+                ('PerJobUserTimeLimit', LARGE_INTEGER),
+                ('LimitFlags', DWORD),
+                ('MinimumWorkingSetSize', SIZE_T),
+                ('MaximumWorkingSetSize', SIZE_T),
+                ('ActiveProcessLimit', DWORD),
+                ('Affinity', ULONG_PTR),
+                ('PriorityClass', DWORD),
+                ('SchedulingClass', DWORD)
+                ]
+
+class JOBOBJECT_ASSOCIATE_COMPLETION_PORT(Structure):
+    _fields_ = [('CompletionKey', c_ulong),
+                ('CompletionPort', HANDLE)]
+
+# see http://msdn.microsoft.com/en-us/library/ms684156%28VS.85%29.aspx
+class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(Structure):
+    _fields_ = [('BasicLimitInformation', JOBOBJECT_BASIC_LIMIT_INFORMATION),
+                ('IoInfo', IO_COUNTERS),
+                ('ProcessMemoryLimit', SIZE_T),
+                ('JobMemoryLimit', SIZE_T),
+                ('PeakProcessMemoryUsed', SIZE_T),
+                ('PeakJobMemoryUsed', SIZE_T)]
+
+# These numbers below come from:
+# http://msdn.microsoft.com/en-us/library/ms686216%28v=vs.85%29.aspx
+JobObjectAssociateCompletionPortInformation = 7
+JobObjectBasicAndIoAccountingInformation = 8
+JobObjectExtendedLimitInformation = 9
+
+class JobObjectInfo(object):
+    mapping = { 'JobObjectBasicAndIoAccountingInformation': 8,
+                'JobObjectExtendedLimitInformation': 9,
+                'JobObjectAssociateCompletionPortInformation': 7
+                }
+    structures = {
+                   7: JOBOBJECT_ASSOCIATE_COMPLETION_PORT,
+                   8: JOBOBJECT_BASIC_AND_IO_ACCOUNTING_INFORMATION,
+                   9: JOBOBJECT_EXTENDED_LIMIT_INFORMATION
+                   }
+    def __init__(self, _class):
+        if isinstance(_class, basestring):
+            assert _class in self.mapping, 'Class should be one of %s; you gave %s' % (self.mapping, _class)
+            _class = self.mapping[_class]
+        assert _class in self.structures, 'Class should be one of %s; you gave %s' % (self.structures, _class)
+        self.code = _class
+        self.info = self.structures[_class]()
+
+
+QueryInformationJobObjectProto = WINFUNCTYPE(
+    BOOL,        # Return type
+    HANDLE,      # hJob
+    DWORD,       # JobObjectInfoClass
+    LPVOID,      # lpJobObjectInfo
+    DWORD,       # cbJobObjectInfoLength
+    LPDWORD      # lpReturnLength
+    )
+
+QueryInformationJobObjectFlags = (
+    (1, 'hJob'),
+    (1, 'JobObjectInfoClass'),
+    (1, 'lpJobObjectInfo'),
+    (1, 'cbJobObjectInfoLength'),
+    (1, 'lpReturnLength', None)
+    )
+
+_QueryInformationJobObject = QueryInformationJobObjectProto(
+    ('QueryInformationJobObject', windll.kernel32),
+    QueryInformationJobObjectFlags
+    )
+
+class SubscriptableReadOnlyStruct(object):
+    def __init__(self, struct):
+        self._struct = struct
+
+    def _delegate(self, name):
+        result = getattr(self._struct, name)
+        if isinstance(result, Structure):
+            return SubscriptableReadOnlyStruct(result)
+        return result
+
+    def __getitem__(self, name):
+        match = [fname for fname, ftype in self._struct._fields_
+                 if fname == name]
+        if match:
+            return self._delegate(name)
+        raise KeyError(name)
+
+    def __getattr__(self, name):
+        return self._delegate(name)
+
+def QueryInformationJobObject(hJob, JobObjectInfoClass):
+    jobinfo = JobObjectInfo(JobObjectInfoClass)
+    result = _QueryInformationJobObject(
+        hJob=hJob,
+        JobObjectInfoClass=jobinfo.code,
+        lpJobObjectInfo=addressof(jobinfo.info),
+        cbJobObjectInfoLength=sizeof(jobinfo.info)
+        )
+    if not result:
+        raise WinError()
+    return SubscriptableReadOnlyStruct(jobinfo.info)

testing/mozbase/mozprocess/mozprocess/winprocess.py

@@ -0,0 +1,457 @@
+# A module to expose various thread/process/job related structures and
+# methods from kernel32
+#
+# The MIT License
+#
+# Copyright (c) 2003-2004 by Peter Astrand <astrand@lysator.liu.se>
+#
+# Additions and modifications written by Benjamin Smedberg
+# <benjamin@smedbergs.us> are Copyright (c) 2006 by the Mozilla Foundation
+# <http://www.mozilla.org/>
+#
+# More Modifications
+# Copyright (c) 2006-2007 by Mike Taylor <bear@code-bear.com>
+# Copyright (c) 2007-2008 by Mikeal Rogers <mikeal@mozilla.com>
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of the
+# author not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
+# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from ctypes import c_void_p, POINTER, sizeof, Structure, Union, windll, WinError, WINFUNCTYPE, c_ulong
+from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LPCWSTR, LPWSTR, UINT, WORD, ULONG
+from qijo import QueryInformationJobObject
+
+LPVOID = c_void_p
+LPBYTE = POINTER(BYTE)
+LPDWORD = POINTER(DWORD)
+LPBOOL = POINTER(BOOL)
+LPULONG = POINTER(c_ulong)
+
+def ErrCheckBool(result, func, args):
+    """errcheck function for Windows functions that return a BOOL True
+    on success"""
+    if not result:
+        raise WinError()
+    return args
+
+
+# AutoHANDLE
+
+class AutoHANDLE(HANDLE):
+    """Subclass of HANDLE which will call CloseHandle() on deletion."""
+    
+    CloseHandleProto = WINFUNCTYPE(BOOL, HANDLE)
+    CloseHandle = CloseHandleProto(("CloseHandle", windll.kernel32))
+    CloseHandle.errcheck = ErrCheckBool
+    
+    def Close(self):
+        if self.value and self.value != HANDLE(-1).value:
+            self.CloseHandle(self)
+            self.value = 0
+    
+    def __del__(self):
+        self.Close()
+
+    def __int__(self):
+        return self.value
+
+def ErrCheckHandle(result, func, args):
+    """errcheck function for Windows functions that return a HANDLE."""
+    if not result:
+        raise WinError()
+    return AutoHANDLE(result)
+
+# PROCESS_INFORMATION structure
+
+class PROCESS_INFORMATION(Structure):
+    _fields_ = [("hProcess", HANDLE),
+                ("hThread", HANDLE),
+                ("dwProcessID", DWORD),
+                ("dwThreadID", DWORD)]
+
+    def __init__(self):
+        Structure.__init__(self)
+        
+        self.cb = sizeof(self)
+
+LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)
+
+# STARTUPINFO structure
+
+class STARTUPINFO(Structure):
+    _fields_ = [("cb", DWORD),
+                ("lpReserved", LPWSTR),
+                ("lpDesktop", LPWSTR),
+                ("lpTitle", LPWSTR),
+                ("dwX", DWORD),
+                ("dwY", DWORD),
+                ("dwXSize", DWORD),
+                ("dwYSize", DWORD),
+                ("dwXCountChars", DWORD),
+                ("dwYCountChars", DWORD),
+                ("dwFillAttribute", DWORD),
+                ("dwFlags", DWORD),
+                ("wShowWindow", WORD),
+                ("cbReserved2", WORD),
+                ("lpReserved2", LPBYTE),
+                ("hStdInput", HANDLE),
+                ("hStdOutput", HANDLE),
+                ("hStdError", HANDLE)
+                ]
+LPSTARTUPINFO = POINTER(STARTUPINFO)
+
+SW_HIDE                 = 0
+
+STARTF_USESHOWWINDOW    = 0x01
+STARTF_USESIZE          = 0x02
+STARTF_USEPOSITION      = 0x04
+STARTF_USECOUNTCHARS    = 0x08
+STARTF_USEFILLATTRIBUTE = 0x10
+STARTF_RUNFULLSCREEN    = 0x20
+STARTF_FORCEONFEEDBACK  = 0x40
+STARTF_FORCEOFFFEEDBACK = 0x80
+STARTF_USESTDHANDLES    = 0x100
+
+# EnvironmentBlock
+
+class EnvironmentBlock:
+    """An object which can be passed as the lpEnv parameter of CreateProcess.
+    It is initialized with a dictionary."""
+
+    def __init__(self, dict):
+        if not dict:
+            self._as_parameter_ = None
+        else:
+            values = ["%s=%s" % (key, value)
+                      for (key, value) in dict.iteritems()]
+            values.append("")
+            self._as_parameter_ = LPCWSTR("\0".join(values))
+
+# Error Messages we need to watch for go here
+# See: http://msdn.microsoft.com/en-us/library/ms681388%28v=vs.85%29.aspx
+ERROR_ABANDONED_WAIT_0 = 735
+            
+# GetLastError()
+GetLastErrorProto = WINFUNCTYPE(DWORD                   # Return Type
+                               )
+GetLastErrorFlags = ()
+GetLastError = GetLastErrorProto(("GetLastError", windll.kernel32), GetLastErrorFlags)
+
+# CreateProcess()
+
+CreateProcessProto = WINFUNCTYPE(BOOL,                  # Return type
+                                 LPCWSTR,               # lpApplicationName
+                                 LPWSTR,                # lpCommandLine
+                                 LPVOID,                # lpProcessAttributes
+                                 LPVOID,                # lpThreadAttributes
+                                 BOOL,                  # bInheritHandles
+                                 DWORD,                 # dwCreationFlags
+                                 LPVOID,                # lpEnvironment
+                                 LPCWSTR,               # lpCurrentDirectory
+                                 LPSTARTUPINFO,         # lpStartupInfo
+                                 LPPROCESS_INFORMATION  # lpProcessInformation
+                                 )
+
+CreateProcessFlags = ((1, "lpApplicationName", None),
+                      (1, "lpCommandLine"),
+                      (1, "lpProcessAttributes", None),
+                      (1, "lpThreadAttributes", None),
+                      (1, "bInheritHandles", True),
+                      (1, "dwCreationFlags", 0),
+                      (1, "lpEnvironment", None),
+                      (1, "lpCurrentDirectory", None),
+                      (1, "lpStartupInfo"),
+                      (2, "lpProcessInformation"))
+
+def ErrCheckCreateProcess(result, func, args):
+    ErrCheckBool(result, func, args)
+    # return a tuple (hProcess, hThread, dwProcessID, dwThreadID)
+    pi = args[9]
+    return AutoHANDLE(pi.hProcess), AutoHANDLE(pi.hThread), pi.dwProcessID, pi.dwThreadID
+
+CreateProcess = CreateProcessProto(("CreateProcessW", windll.kernel32),
+                                   CreateProcessFlags)
+CreateProcess.errcheck = ErrCheckCreateProcess
+
+# flags for CreateProcess
+CREATE_BREAKAWAY_FROM_JOB = 0x01000000
+CREATE_DEFAULT_ERROR_MODE = 0x04000000
+CREATE_NEW_CONSOLE = 0x00000010
+CREATE_NEW_PROCESS_GROUP = 0x00000200
+CREATE_NO_WINDOW = 0x08000000
+CREATE_SUSPENDED = 0x00000004
+CREATE_UNICODE_ENVIRONMENT = 0x00000400
+
+# Flags for IOCompletion ports (some of these would probably be defined if 
+# we used the win32 extensions for python, but we don't want to do that if we 
+# can help it.
+INVALID_HANDLE_VALUE = HANDLE(-1) # From winbase.h
+
+# Self Defined Constants for IOPort <--> Job Object communication
+COMPKEY_TERMINATE = c_ulong(0)
+COMPKEY_JOBOBJECT = c_ulong(1)
+
+# flags for job limit information
+# see http://msdn.microsoft.com/en-us/library/ms684147%28VS.85%29.aspx
+JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x00000800
+JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK = 0x00001000
+
+# Flags for Job Object Completion Port Message IDs from winnt.h
+# See also: http://msdn.microsoft.com/en-us/library/ms684141%28v=vs.85%29.aspx
+JOB_OBJECT_MSG_END_OF_JOB_TIME =          1
+JOB_OBJECT_MSG_END_OF_PROCESS_TIME =      2
+JOB_OBJECT_MSG_ACTIVE_PROCESS_LIMIT =     3
+JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO =      4
+JOB_OBJECT_MSG_NEW_PROCESS =              6
+JOB_OBJECT_MSG_EXIT_PROCESS =             7
+JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS =    8
+JOB_OBJECT_MSG_PROCESS_MEMORY_LIMIT =     9
+JOB_OBJECT_MSG_JOB_MEMORY_LIMIT =         10
+
+# See winbase.h
+DEBUG_ONLY_THIS_PROCESS = 0x00000002
+DEBUG_PROCESS = 0x00000001
+DETACHED_PROCESS = 0x00000008
+    
+# GetQueuedCompletionPortStatus - http://msdn.microsoft.com/en-us/library/aa364986%28v=vs.85%29.aspx
+GetQueuedCompletionStatusProto = WINFUNCTYPE(BOOL,         # Return Type
+                                             HANDLE,       # Completion Port
+                                             LPDWORD,      # Msg ID
+                                             LPULONG,      # Completion Key
+                                             LPULONG,      # PID Returned from the call (may be null)
+                                             DWORD)        # milliseconds to wait
+GetQueuedCompletionStatusFlags = ((1, "CompletionPort", INVALID_HANDLE_VALUE),
+                                  (1, "lpNumberOfBytes", None),
+                                  (1, "lpCompletionKey", None),
+                                  (1, "lpPID", None),
+                                  (1, "dwMilliseconds", 0))
+GetQueuedCompletionStatus = GetQueuedCompletionStatusProto(("GetQueuedCompletionStatus",
+                                                            windll.kernel32),
+                                                           GetQueuedCompletionStatusFlags)
+
+# CreateIOCompletionPort
+# Note that the completion key is just a number, not a pointer.
+CreateIoCompletionPortProto = WINFUNCTYPE(HANDLE,      # Return Type
+                                          HANDLE,      # File Handle
+                                          HANDLE,      # Existing Completion Port
+                                          c_ulong,     # Completion Key
+                                          DWORD        # Number of Threads
+                                         )
+CreateIoCompletionPortFlags = ((1, "FileHandle", INVALID_HANDLE_VALUE),
+                               (1, "ExistingCompletionPort", None),
+                               (1, "CompletionKey", c_ulong(0)),
+                               (1, "NumberOfConcurrentThreads", 0))
+CreateIoCompletionPort = CreateIoCompletionPortProto(("CreateIoCompletionPort",
+                                                      windll.kernel32),
+                                                      CreateIoCompletionPortFlags)
+CreateIoCompletionPort.errcheck = ErrCheckHandle
+
+# SetInformationJobObject
+SetInformationJobObjectProto = WINFUNCTYPE(BOOL,      # Return Type
+                                           HANDLE,    # Job Handle
+                                           DWORD,     # Type of Class next param is
+                                           LPVOID,    # Job Object Class
+                                           DWORD      # Job Object Class Length
+                                          )
+SetInformationJobObjectProtoFlags = ((1, "hJob", None),
+                                     (1, "JobObjectInfoClass", None),
+                                     (1, "lpJobObjectInfo", None),
+                                     (1, "cbJobObjectInfoLength", 0))
+SetInformationJobObject = SetInformationJobObjectProto(("SetInformationJobObject",
+                                                        windll.kernel32),
+                                                        SetInformationJobObjectProtoFlags)
+SetInformationJobObject.errcheck = ErrCheckBool
+
+# CreateJobObject()
+CreateJobObjectProto = WINFUNCTYPE(HANDLE,             # Return type
+                                   LPVOID,             # lpJobAttributes
+                                   LPCWSTR             # lpName
+                                   )
+
+CreateJobObjectFlags = ((1, "lpJobAttributes", None),
+                        (1, "lpName", None))
+
+CreateJobObject = CreateJobObjectProto(("CreateJobObjectW", windll.kernel32),
+                                       CreateJobObjectFlags)
+CreateJobObject.errcheck = ErrCheckHandle
+
+# AssignProcessToJobObject()
+
+AssignProcessToJobObjectProto = WINFUNCTYPE(BOOL,      # Return type
+                                            HANDLE,    # hJob
+                                            HANDLE     # hProcess
+                                            )
+AssignProcessToJobObjectFlags = ((1, "hJob"),
+                                 (1, "hProcess"))
+AssignProcessToJobObject = AssignProcessToJobObjectProto(
+    ("AssignProcessToJobObject", windll.kernel32),
+    AssignProcessToJobObjectFlags)
+AssignProcessToJobObject.errcheck = ErrCheckBool
+
+# GetCurrentProcess()
+# because os.getPid() is way too easy
+GetCurrentProcessProto = WINFUNCTYPE(HANDLE    # Return type
+                                     )
+GetCurrentProcessFlags = ()
+GetCurrentProcess = GetCurrentProcessProto(
+    ("GetCurrentProcess", windll.kernel32),
+    GetCurrentProcessFlags)
+GetCurrentProcess.errcheck = ErrCheckHandle
+
+# IsProcessInJob()
+try:
+    IsProcessInJobProto = WINFUNCTYPE(BOOL,     # Return type
+                                      HANDLE,   # Process Handle
+                                      HANDLE,   # Job Handle
+                                      LPBOOL      # Result
+                                      )
+    IsProcessInJobFlags = ((1, "ProcessHandle"),
+                           (1, "JobHandle", HANDLE(0)),
+                           (2, "Result"))
+    IsProcessInJob = IsProcessInJobProto(
+        ("IsProcessInJob", windll.kernel32),
+        IsProcessInJobFlags)
+    IsProcessInJob.errcheck = ErrCheckBool 
+except AttributeError:
+    # windows 2k doesn't have this API
+    def IsProcessInJob(process):
+        return False
+
+
+# ResumeThread()
+
+def ErrCheckResumeThread(result, func, args):
+    if result == -1:
+        raise WinError()
+
+    return args
+
+ResumeThreadProto = WINFUNCTYPE(DWORD,      # Return type
+                                HANDLE      # hThread
+                                )
+ResumeThreadFlags = ((1, "hThread"),)
+ResumeThread = ResumeThreadProto(("ResumeThread", windll.kernel32),
+                                 ResumeThreadFlags)
+ResumeThread.errcheck = ErrCheckResumeThread
+
+# TerminateProcess()
+
+TerminateProcessProto = WINFUNCTYPE(BOOL,   # Return type
+                                    HANDLE, # hProcess
+                                    UINT    # uExitCode
+                                    )
+TerminateProcessFlags = ((1, "hProcess"),
+                         (1, "uExitCode", 127))
+TerminateProcess = TerminateProcessProto(
+    ("TerminateProcess", windll.kernel32),
+    TerminateProcessFlags)
+TerminateProcess.errcheck = ErrCheckBool
+
+# TerminateJobObject()
+
+TerminateJobObjectProto = WINFUNCTYPE(BOOL,   # Return type
+                                      HANDLE, # hJob
+                                      UINT    # uExitCode
+                                      )
+TerminateJobObjectFlags = ((1, "hJob"),
+                           (1, "uExitCode", 127))
+TerminateJobObject = TerminateJobObjectProto(
+    ("TerminateJobObject", windll.kernel32),
+    TerminateJobObjectFlags)
+TerminateJobObject.errcheck = ErrCheckBool
+
+# WaitForSingleObject()
+
+WaitForSingleObjectProto = WINFUNCTYPE(DWORD,  # Return type
+                                       HANDLE, # hHandle
+                                       DWORD,  # dwMilliseconds
+                                       )
+WaitForSingleObjectFlags = ((1, "hHandle"),
+                            (1, "dwMilliseconds", -1))
+WaitForSingleObject = WaitForSingleObjectProto(
+    ("WaitForSingleObject", windll.kernel32),
+    WaitForSingleObjectFlags)
+
+# http://msdn.microsoft.com/en-us/library/ms681381%28v=vs.85%29.aspx
+INFINITE = -1
+WAIT_TIMEOUT = 0x0102
+WAIT_OBJECT_0 = 0x0
+WAIT_ABANDONED = 0x0080
+
+# http://msdn.microsoft.com/en-us/library/ms683189%28VS.85%29.aspx
+STILL_ACTIVE = 259
+
+# Used when we terminate a process.
+ERROR_CONTROL_C_EXIT = 0x23c
+
+# GetExitCodeProcess()
+
+GetExitCodeProcessProto = WINFUNCTYPE(BOOL,    # Return type
+                                      HANDLE,  # hProcess
+                                      LPDWORD, # lpExitCode
+                                      )
+GetExitCodeProcessFlags = ((1, "hProcess"),
+                           (2, "lpExitCode"))
+GetExitCodeProcess = GetExitCodeProcessProto(
+    ("GetExitCodeProcess", windll.kernel32),
+    GetExitCodeProcessFlags)
+GetExitCodeProcess.errcheck = ErrCheckBool
+
+def CanCreateJobObject():
+    currentProc = GetCurrentProcess()
+    if IsProcessInJob(currentProc):
+        jobinfo = QueryInformationJobObject(HANDLE(0), 'JobObjectExtendedLimitInformation')
+        limitflags = jobinfo['BasicLimitInformation']['LimitFlags']
+        return bool(limitflags & JOB_OBJECT_LIMIT_BREAKAWAY_OK) or bool(limitflags & JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK)
+    else:
+        return True
+
+### testing functions
+
+def parent():
+    print 'Starting parent'
+    currentProc = GetCurrentProcess()
+    if IsProcessInJob(currentProc):
+        print >> sys.stderr, "You should not be in a job object to test"
+        sys.exit(1)
+    assert CanCreateJobObject()
+    print 'File: %s' % __file__
+    command = [sys.executable, __file__, '-child']
+    print 'Running command: %s' % command
+    process = Popen(command)
+    process.kill()
+    code = process.returncode
+    print 'Child code: %s' % code
+    assert code == 127
+        
+def child():
+    print 'Starting child'
+    currentProc = GetCurrentProcess()
+    injob = IsProcessInJob(currentProc)
+    print "Is in a job?: %s" % injob
+    can_create = CanCreateJobObject()
+    print 'Can create job?: %s' % can_create
+    process = Popen('c:\\windows\\notepad.exe')
+    assert process._job
+    jobinfo = QueryInformationJobObject(process._job, 'JobObjectExtendedLimitInformation')
+    print 'Job info: %s' % jobinfo
+    limitflags = jobinfo['BasicLimitInformation']['LimitFlags']
+    print 'LimitFlags: %s' % limitflags
+    process.kill()

testing/mozbase/mozprocess/mozprocess/wpk.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprocess.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Clint Talbert <ctalbert@mozilla.com>
+#  Jonathan Griffin <jgriffin@mozilla.com>
+#  Jeff Hammel <jhammel@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+from ctypes import sizeof, windll, addressof, c_wchar, create_unicode_buffer
+from ctypes.wintypes import DWORD, HANDLE
+
+PROCESS_TERMINATE = 0x0001
+PROCESS_QUERY_INFORMATION = 0x0400
+PROCESS_VM_READ = 0x0010
+
+def get_pids(process_name):
+    BIG_ARRAY = DWORD * 4096
+    processes = BIG_ARRAY()
+    needed = DWORD()
+
+    pids = []
+    result = windll.psapi.EnumProcesses(processes,
+                                        sizeof(processes),
+                                        addressof(needed))
+    if not result:
+        return pids
+
+    num_results = needed.value / sizeof(DWORD)
+
+    for i in range(num_results):
+        pid = processes[i]
+        process = windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION |
+                                              PROCESS_VM_READ,
+                                              0, pid)
+        if process:
+            module = HANDLE()
+            result = windll.psapi.EnumProcessModules(process,
+                                                     addressof(module),
+                                                     sizeof(module),
+                                                     addressof(needed))
+            if result:
+                name = create_unicode_buffer(1024)
+                result = windll.psapi.GetModuleBaseNameW(process, module,
+                                                         name, len(name))
+                # TODO: This might not be the best way to
+                # match a process name; maybe use a regexp instead.
+                if name.value.startswith(process_name):
+                    pids.append(pid)
+                windll.kernel32.CloseHandle(module)
+            windll.kernel32.CloseHandle(process)
+
+    return pids
+
+def kill_pid(pid):
+    process = windll.kernel32.OpenProcess(PROCESS_TERMINATE, 0, pid)
+    if process:
+        windll.kernel32.TerminateProcess(process, 0)
+        windll.kernel32.CloseHandle(process)
+
+if __name__ == '__main__':
+    import subprocess
+    import time
+
+    # This test just opens a new notepad instance and kills it.
+
+    name = 'notepad'
+
+    old_pids = set(get_pids(name))
+    subprocess.Popen([name])
+    time.sleep(0.25)
+    new_pids = set(get_pids(name)).difference(old_pids)
+
+    if len(new_pids) != 1:
+        raise Exception('%s was not opened or get_pids() is '
+                        'malfunctioning' % name)
+
+    kill_pid(tuple(new_pids)[0])
+
+    newest_pids = set(get_pids(name)).difference(old_pids)
+
+    if len(newest_pids) != 0:
+        raise Exception('kill_pid() is malfunctioning')
+
+    print "Test passed."

testing/mozbase/mozprocess/setup.py

@@ -0,0 +1,69 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprocess.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Clint Talbert <ctalbert@mozilla.com>
+#  Jonathan Griffin <jgriffin@mozilla.com>
+#  Jeff Hammel <jhammel@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os
+from setuptools import setup, find_packages
+
+version = '0.1b2'
+
+# take description from README
+here = os.path.dirname(os.path.abspath(__file__))
+try:
+    description = file(os.path.join(here, 'README.md')).read()
+except (OSError, IOError):
+    description = ''
+
+setup(name='mozprocess',
+      version=version,
+      description="Mozilla-authored process handling",
+      long_description=description,
+      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+      keywords='',
+      author='Mozilla Automation and Testing Team',
+      author_email='mozmill-dev@googlegroups.com',
+      url='http://github.com/mozautomation/mozmill',
+      license='MPL',
+      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
+      include_package_data=True,
+      zip_safe=False,
+      install_requires=['mozinfo'],
+      entry_points="""
+      # -*- Entry points: -*-
+      """,
+      )

testing/mozbase/mozprofile/README.md

@@ -0,0 +1,80 @@
+[Mozprofile](https://github.com/mozilla/mozbase/tree/master/mozprofile)
+is a python tool for creating and managing profiles for Mozilla's
+applications (Firefox, Thunderbird, etc.). In addition to creating profiles,
+mozprofile can install [addons](https://developer.mozilla.org/en/addons)
+and set [preferences](https://developer.mozilla.org/En/A_Brief_Guide_to_Mozilla_Preferences).  
+Mozprofile can be utilized from the command line or as an API.
+
+
+# Command Line Usage
+
+mozprofile may be used to create profiles, set preferences in
+profiles, or install addons into profiles.
+
+The profile to be operated on may be specified with the `--profile`
+switch. If a profile is not specified, one will be created in a
+temporary directory which will be echoed to the terminal:
+
+    (mozmill)> mozprofile 
+    /tmp/tmp4q1iEU.mozrunner
+    (mozmill)> ls /tmp/tmp4q1iEU.mozrunner
+    user.js
+
+To run mozprofile from the command line enter:
+`mozprofile --help` for a list of options.
+
+
+# API Usage
+
+To use mozprofile as an API you can import
+[mozprofile.profile](https://github.com/mozilla/mozbase/tree/master/mozprofile/mozprofile/profile.py)
+and/or the
+[AddonManager](https://github.com/mozilla/mozbase/tree/master/mozprofile/mozprofile/addons.py). 
+
+`mozprofile.profile` features a generic `Profile` class.  In addition,
+subclasses `FirefoxProfile` and `ThundebirdProfile` are available
+with preset preferences for those applications.
+
+
+# Installing Addons
+
+Addons may be installed individually or from a manifest.
+
+Example:
+
+	from mozprofile import FirefoxProfile
+	
+	# create new profile to pass to mozmill/mozrunner
+	profile = FirefoxProfile(addons=["adblock.xpi"])
+
+
+# Setting Preferences
+
+Preferences can be set in several ways:
+
+- using the API: You can pass preferences in to the Profile class's
+  constructor: `obj = FirefoxProfile(preferences=[("accessibility.typeaheadfind.flashBar", 0)])`
+- using a JSON blob file: `mozprofile --preferences myprefs.json`
+- using a `.ini` file: `mozprofile --preferences myprefs.ini`
+- via the command line: `mozprofile --pref key:value --pref key:value [...]`
+
+When setting preferences from  an `.ini` file or the `--pref` switch,
+the value will be interpolated as an integer or a boolean
+(`true`/`false`) if possible.
+
+# Setting Permissions
+
+mozprofile also takes care of adding permissions to the profile.
+See https://github.com/mozilla/mozbase/blob/master/mozprofile/mozprofile/permissions.py
+
+
+# Resources
+
+Other Mozilla programs offer additional and overlapping functionality
+for profiles.  There is also substantive documentation on profiles and
+their management.
+
+- [ProfileManager](https://developer.mozilla.org/en/Profile_Manager) : 
+  XULRunner application for managing profiles. Has a GUI and CLI.
+- [python-profilemanager](http://k0s.org/mozilla/hg/profilemanager/) : python CLI interface similar to ProfileManager
+- profile documentation : http://support.mozilla.com/en-US/kb/Profiles

testing/mozbase/mozprofile/mozprofile/__init__.py

@@ -0,0 +1,43 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprofile.
+#
+# The Initial Developer of the Original Code is
+#  The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2008-2009
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Mikeal Rogers <mikeal.rogers@gmail.com>
+#  Clint Talbert <ctalbert@mozilla.com>
+#  Henrik Skupin <hskupin@mozilla.com>
+#  Jeff Hammel <jhammel@mozilla.com>
+#  Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+from profile import *
+from addons import *
+from cli import *

testing/mozbase/mozprofile/mozprofile/addons.py

@@ -0,0 +1,261 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozprofile.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Andrew Halberstadt <halbersa@gmail.com>
+#   Mikeal Rogers <mikeal.rogers@gmail.com>
+#   Clint Talbert <ctalbert@mozilla.com>
+#   Jeff Hammel <jhammel@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os
+import shutil
+import sys
+import tempfile
+import urllib2
+import zipfile
+from distutils import dir_util
+from manifestparser import ManifestParser
+from xml.dom import minidom
+
+# Needed for the AMO's rest API - https://developer.mozilla.org/en/addons.mozilla.org_%28AMO%29_API_Developers%27_Guide/The_generic_AMO_API
+AMO_API_VERSION = "1.5"
+
+class AddonManager(object):
+    """
+    Handles all operations regarding addons including: installing and cleaning addons
+    """
+
+    def __init__(self, profile):
+        """
+        profile - the path to the profile for which we install addons
+        """
+        self.profile = profile
+        self.installed_addons = []
+        # keeps track of addons and manifests that were passed to install_addons
+        self.addons = []
+        self.manifests = []
+
+
+    def install_addons(self, addons=None, manifests=None):
+        """
+        Installs all types of addons
+        addons - a list of addon paths to install
+        manifest - a list of addon manifests to install
+        """
+        # install addon paths
+        if addons:
+            if isinstance(addons, basestring):
+                addons = [addons]
+            for addon in addons:
+                self.install_from_path(addon)
+        # install addon manifests
+        if manifests:
+            if isinstance(manifests, basestring):
+                manifests = [manifests]
+            for manifest in manifests:
+                self.install_from_manifest(manifest)
+
+
+    def install_from_manifest(self, filepath):
+        """
+        Installs addons from a manifest
+        filepath - path to the manifest of addons to install
+        """
+        self.manifests.append(filepath)
+        manifest = ManifestParser()
+        manifest.read(filepath)
+        addons = manifest.get()
+
+        for addon in addons:
+            if '://' in addon['path'] or os.path.exists(addon['path']):
+                self.install_from_path(addon['path'])
+                continue
+
+            # No path specified, try to grab it off AMO
+            locale = addon.get('amo_locale', 'en_US')
+
+            query = 'https://services.addons.mozilla.org/' + locale + '/firefox/api/' + AMO_API_VERSION + '/'
+            if 'amo_id' in addon:
+                query += 'addon/' + addon['amo_id']                 # this query grabs information on the addon base on its id
+            else:
+                query += 'search/' + addon['name'] + '/default/1'   # this query grabs information on the first addon returned from a search
+            install_path = AddonManager.get_amo_install_path(query)
+            self.install_from_path(install_path)
+
+    @classmethod
+    def get_amo_install_path(self, query):
+        """
+        Return the addon xpi install path for the specified AMO query.
+        See: https://developer.mozilla.org/en/addons.mozilla.org_%28AMO%29_API_Developers%27_Guide/The_generic_AMO_API
+        for query documentation.
+        """
+        response = urllib2.urlopen(query)
+        dom = minidom.parseString(response.read())
+        for node in dom.getElementsByTagName('install')[0].childNodes:
+            if node.nodeType == node.TEXT_NODE:
+                return node.data
+
+    @classmethod
+    def addon_details(cls, addon_path):
+        """
+        returns a dictionary of details about the addon
+        - addon_path : path to the addon directory
+        Returns:
+        {'id':      u'rainbow@colors.org', # id of the addon
+         'version': u'1.4',                # version of the addon
+         'name':    u'Rainbow',            # name of the addon
+         'unpack': # whether to unpack the addon
+        """
+
+        # TODO: We don't use the unpack variable yet, but we should: bug 662683
+        details = {
+            'id': None,
+            'unpack': False,
+            'name': None,
+            'version': None
+        }
+
+        def get_namespace_id(doc, url):
+            attributes = doc.documentElement.attributes
+            namespace = ""
+            for i in range(attributes.length):
+                if attributes.item(i).value == url:
+                    if ":" in attributes.item(i).name:
+                        # If the namespace is not the default one remove 'xlmns:'
+                        namespace = attributes.item(i).name.split(':')[1] + ":"
+                        break
+            return namespace
+
+        def get_text(element):
+            """Retrieve the text value of a given node"""
+            rc = []
+            for node in element.childNodes:
+                if node.nodeType == node.TEXT_NODE:
+                    rc.append(node.data)
+            return ''.join(rc).strip()
+
+        doc = minidom.parse(os.path.join(addon_path, 'install.rdf'))
+
+        # Get the namespaces abbreviations
+        em = get_namespace_id(doc, "http://www.mozilla.org/2004/em-rdf#")
+        rdf = get_namespace_id(doc, "http://www.w3.org/1999/02/22-rdf-syntax-ns#")
+
+        description = doc.getElementsByTagName(rdf + "Description").item(0)
+        for node in description.childNodes:
+            # Remove the namespace prefix from the tag for comparison
+            entry = node.nodeName.replace(em, "")
+            if entry in details.keys():
+                details.update({ entry: get_text(node) })
+
+        # turn unpack into a true/false value
+        if isinstance(details['unpack'], basestring):
+            details['unpack'] = details['unpack'].lower() == 'true'
+
+        return details
+
+    def install_from_path(self, path, unpack=False):
+        """
+        Installs addon from a filepath, url
+        or directory of addons in the profile.
+        - path: url, path to .xpi, or directory of addons
+        - unpack: whether to unpack unless specified otherwise in the install.rdf
+        """
+        self.addons.append(path)
+
+        # if the addon is a url, download it
+        # note that this won't work with protocols urllib2 doesn't support
+        if '://' in path:
+            response = urllib2.urlopen(path)
+            fd, path = tempfile.mkstemp(suffix='.xpi')
+            os.write(fd, response.read())
+            os.close(fd)
+            tmpfile = path
+        else:
+            tmpfile = None
+
+        # if the addon is a directory, install all addons in it
+        addons = [path]
+        if not path.endswith('.xpi') and not os.path.exists(os.path.join(path, 'install.rdf')):
+            assert os.path.isdir(path), "Addon '%s' cannot be installed" % path
+            addons = [os.path.join(path, x) for x in os.listdir(path)]
+
+        # install each addon
+        for addon in addons:
+            tmpdir = None
+            xpifile = None
+            if addon.endswith('.xpi'):
+                tmpdir = tempfile.mkdtemp(suffix = '.' + os.path.split(addon)[-1])
+                compressed_file = zipfile.ZipFile(addon, 'r')
+                for name in compressed_file.namelist():
+                    if name.endswith('/'):
+                        os.makedirs(os.path.join(tmpdir, name))
+                    else:
+                        if not os.path.isdir(os.path.dirname(os.path.join(tmpdir, name))):
+                            os.makedirs(os.path.dirname(os.path.join(tmpdir, name)))
+                        data = compressed_file.read(name)
+                        f = open(os.path.join(tmpdir, name), 'wb')
+                        f.write(data)
+                        f.close()
+                xpifile = addon
+                addon = tmpdir
+
+            # determine the addon id
+            addon_details = AddonManager.addon_details(addon)
+            addon_id = addon_details.get('id')
+            assert addon_id, 'The addon id could not be found: %s' % addon
+
+            # copy the addon to the profile
+            extensions_path = os.path.join(self.profile, 'extensions')
+            addon_path = os.path.join(extensions_path, addon_id)
+            if not unpack and not addon_details['unpack'] and xpifile:
+                if not os.path.exists(extensions_path):
+                    os.makedirs(extensions_path)
+                shutil.copy(xpifile, addon_path + '.xpi')
+            else:
+                dir_util.copy_tree(addon, addon_path, preserve_symlinks=1)
+                self.installed_addons.append(addon_path)
+
+            # remove the temporary directory, if any
+            if tmpdir:
+                dir_util.remove_tree(tmpdir)
+
+        # remove temporary file, if any
+        if tmpfile:
+            os.remove(tmpfile)
+
+    def clean_addons(self):
+        """Cleans up addons in the profile."""
+        for addon in self.installed_addons:
+            if os.path.isdir(addon):
+                dir_util.remove_tree(addon)

testing/mozbase/mozprofile/mozprofile/cli.py

@@ -0,0 +1,128 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprofile command line interface.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mikeal Rogers <mikeal.rogers@gmail.com>
+#   Clint Talbert <ctalbert@mozilla.com>
+#   Jeff Hammel <jhammel@mozilla.com>
+#   Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""
+Creates and/or modifies a Firefox profile.
+The profile can be modified by passing in addons to install or preferences to set.
+If no profile is specified, a new profile is created and the path of the resulting profile is printed.
+"""
+
+import sys
+from addons import AddonManager
+from optparse import OptionParser
+from prefs import Preferences
+from profile import Profile
+
+__all__ = ['MozProfileCLI', 'cli']
+
+class MozProfileCLI(object):
+
+    module = 'mozprofile'
+
+    def __init__(self, args=sys.argv[1:]):
+        self.parser = OptionParser(description=__doc__)
+        self.add_options(self.parser)
+        (self.options, self.args) = self.parser.parse_args(args)
+
+    def add_options(self, parser):
+
+        parser.add_option("-p", "--profile", dest="profile",
+                          help="The path to the profile to operate on. If none, creates a new profile in temp directory")
+        parser.add_option("-a", "--addon", dest="addons",
+                          action="append", default=[],
+                          help="Addon paths to install. Can be a filepath, a directory containing addons, or a url")
+        parser.add_option("--addon-manifests", dest="addon_manifests",
+                          action="append",
+                          help="An addon manifest to install")
+        parser.add_option("--pref", dest="prefs",
+                          action='append', default=[],
+                          help="A preference to set. Must be a key-value pair separated by a ':'")
+        parser.add_option("--preferences", dest="prefs_files",
+                          action='append', default=[],
+                          metavar="FILE",
+                          help="read preferences from a JSON or INI file. For INI, use 'file.ini:section' to specify a particular section.")
+
+    def profile_args(self):
+        """arguments to instantiate the profile class"""
+        return dict(profile=self.options.profile,
+                    addons=self.options.addons,
+                    addon_manifests=self.options.addon_manifests,
+                    preferences=self.preferences())
+
+    def preferences(self):
+        """profile preferences"""
+
+        # object to hold preferences
+        prefs = Preferences()
+
+        # add preferences files
+        for prefs_file in self.options.prefs_files:
+            prefs.add_file(prefs_file)
+
+        # change CLI preferences into 2-tuples
+        separator = ':'
+        cli_prefs = []
+        for pref in self.options.prefs:
+            if separator not in pref:
+                self.parser.error("Preference must be a key-value pair separated by a ':' (You gave: %s)" % pref)
+            cli_prefs.append(pref.split(separator, 1))
+
+        # string preferences
+        prefs.add(cli_prefs, cast=True)
+
+        return prefs()
+
+
+def cli(args=sys.argv[1:]):
+
+    # process the command line
+    cli = MozProfileCLI(args)
+
+    # create the profile
+    kwargs = cli.profile_args()
+    kwargs['restore'] = False
+    profile = Profile(**kwargs)
+
+    # if no profile was passed in print the newly created profile
+    if not cli.options.profile:
+        print profile.profile
+
+if __name__ == '__main__':
+    cli()

testing/mozbase/mozprofile/mozprofile/permissions.py

@@ -0,0 +1,316 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozprofile.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Joel Maher <joel.maher@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+
+"""
+add permissions to the profile
+"""
+
+__all__ = ['LocationsSyntaxError', 'Location', 'PermissionsManager']
+
+import codecs
+import itertools
+import os
+import sqlite3
+import urlparse
+
+class LocationsSyntaxError(Exception):
+    "Signifies a syntax error on a particular line in server-locations.txt."
+
+    def __init__(self, lineno, msg = None):
+        self.lineno = lineno
+        self.msg = msg
+
+    def __str__(self):
+        s = "Syntax error on line %s" % self.lineno
+        if self.msg:
+            s += ": %s." % self.msg
+        else:
+            s += "."
+        return s
+
+
+class Location(object):
+    "Represents a location line in server-locations.txt."
+
+    attrs = ('scheme', 'host', 'port')
+
+    def __init__(self, scheme, host, port, options):
+        for attr in self.attrs:
+            setattr(self, attr, locals()[attr])
+        self.options = options
+
+    def isEqual(self, location):
+        "compare scheme://host:port, but ignore options"
+        return len([i for i in self.attrs if getattr(self, i) == getattr(location, i)]) == len(self.attrs)
+
+    __eq__ = isEqual
+
+    def url(self):
+        return '%s://%s:%s' % (self.scheme, self.host, self.port)
+
+    def __str__(self):
+        return  '%s  %s' % (self.url(), ','.join(self.options))
+
+
+class PermissionsManager(object):
+    _num_permissions = 0
+
+    def __init__(self, profileDir, locations=None):
+        self._profileDir = profileDir
+        self._locations = [] # for cleanup
+        if locations:
+            if isinstance(locations, list):
+                for l in locations:
+                    self.add_host(**l)
+            elif isinstance(locations, dict):
+                self.add_host(**locations)
+            elif os.path.exists(locations):
+                self.add_file(locations)
+
+    def write_permission(self, location):
+        """write permissions to the sqlite database"""
+
+        # Open database and create table
+        permDB = sqlite3.connect(os.path.join(self._profileDir, "permissions.sqlite"))
+        cursor = permDB.cursor();
+        # SQL copied from
+        # http://mxr.mozilla.org/mozilla-central/source/extensions/cookie/nsPermissionManager.cpp
+        cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (
+           id INTEGER PRIMARY KEY,
+           host TEXT,
+           type TEXT,
+           permission INTEGER,
+           expireType INTEGER,
+           expireTime INTEGER)""")
+
+        # set the permissions
+        permissions = {'allowXULXBL':[(location.host, 'noxul' not in location.options)]}
+        for perm in permissions.keys():
+            for host,allow in permissions[perm]:
+                self._num_permissions += 1
+                cursor.execute("INSERT INTO moz_hosts values(?, ?, ?, ?, 0, 0)",
+                               (self._num_permissions, host, perm, 1 if allow else 2))
+
+        # Commit and close
+        permDB.commit()
+        cursor.close()
+
+    def add(self, *newLocations):
+        """add locations to the database"""
+
+        for location in newLocations:
+            for loc in self._locations:
+                if loc.isEqual(location):
+                    print >> sys.stderr, "Duplicate location: %s" % location.url()
+                    break
+        else:
+            self._locations.append(location)
+            self.write_permission(location)
+
+    def add_host(self, host, port='80', scheme='http', options='privileged'):
+        if isinstance(options, basestring):
+            options = options.split(',')
+        self.add(Location(scheme, host, port, options))
+
+    def add_file(self, path):
+        """add permissions from a locations file """
+        self.add(self.read_locations(path))
+
+    def read_locations(self, filename):
+        """
+        Reads the file (in the format of server-locations.txt) and add all
+        valid locations to the self.locations array.
+
+        This format:
+        http://mxr.mozilla.org/mozilla-central/source/build/pgo/server-locations.txt
+        """
+
+        locationFile = codecs.open(filename, "r", "UTF-8")
+
+        locations = []
+        lineno = 0
+        seenPrimary = False
+        for line in locationFile:
+            line = line.strip()
+            lineno += 1
+
+            # check for comments and blank lines
+            if line.startswith("#") or not line:
+                continue
+
+            # split the server from the options
+            try:
+                server, options = line.rsplit(None, 1)
+                options = options.split(',')
+            except ValueError:
+                server = line
+                options = []
+
+            # parse the server url
+            if '://' not in server:
+                server = 'http://' + server
+            scheme, netloc, path, query, fragment = urlparse.urlsplit(server)
+            # get the host and port
+            try:
+                host, port = netloc.rsplit(':', 1)
+            except ValueError:
+                host = netloc
+                port = '80'
+            try:
+                int(port)
+            except ValueError:
+                raise LocationsSyntaxError(lineno, 'bad value for port: %s' % line)
+
+            # check for primary location
+            if "primary" in options:
+                if seenPrimary:
+                    raise LocationsSyntaxError(lineno, "multiple primary locations")
+                seenPrimary = True
+
+            # add the location
+            locations.append(Location(scheme, host, port, options))
+
+        # ensure that a primary is found
+        if not seenPrimary:
+            raise LocationsSyntaxError(lineno + 1, "missing primary location")
+
+        return locations
+
+    def getNetworkPreferences(self, proxy=False):
+        """
+        take known locations and generate preferences to handle permissions and proxy
+        returns a tuple of prefs, user_prefs
+        """
+
+        # Grant God-power to all the privileged servers on which tests run.
+        prefs = []
+        privileged = filter(lambda loc: "privileged" in loc.options, self._locations)
+        for (i, l) in itertools.izip(itertools.count(1), privileged):
+            prefs.append(("capability.principal.codebase.p%s.granted" % i, "UniversalPreferencesWrite UniversalXPConnect UniversalPreferencesRead"))
+
+            # TODO: do we need the port?
+            prefs.append(("capability.principal.codebase.p%s.id" % i, l.scheme + "://" + l.host))
+            prefs.append(("capability.principal.codebase.p%s.subjectName" % i, ""))
+
+        if proxy:
+            user_prefs = self.pacPrefs()
+        else:
+            user_prefs = []
+
+        return prefs, user_prefs
+
+    def pacPrefs(self):
+        """
+        return preferences for Proxy Auto Config. originally taken from
+        http://mxr.mozilla.org/mozilla-central/source/build/automation.py.in
+        """
+
+        prefs = []
+
+        # We need to proxy every server but the primary one.
+        origins = ["'%s'" % l.url()
+                   for l in self._locations
+                   if "primary" not in l.options]
+        origins = ", ".join(origins)
+
+        # TODO: this is not a reliable way to determine the Proxy host
+        for l in self._locations:
+            if "primary" in l.options:
+                webServer = l.host
+                httpPort  = l.port
+                sslPort   = 443
+
+        # TODO: this should live in a template!
+        pacURL = """data:text/plain,
+function FindProxyForURL(url, host)
+{
+  var origins = [%(origins)s];
+  var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
+                         '://' +
+                         '(?:[^/@]*@)?' +
+                         '(.*?)' +
+                         '(?::(\\\\\\\\d+))?/');
+  var matches = regex.exec(url);
+  if (!matches)
+    return 'DIRECT';
+  var isHttp = matches[1] == 'http';
+  var isHttps = matches[1] == 'https';
+  var isWebSocket = matches[1] == 'ws';
+  var isWebSocketSSL = matches[1] == 'wss';
+  if (!matches[3])
+  {
+    if (isHttp | isWebSocket) matches[3] = '80';
+    if (isHttps | isWebSocketSSL) matches[3] = '443';
+  }
+  if (isWebSocket)
+    matches[1] = 'http';
+  if (isWebSocketSSL)
+    matches[1] = 'https';
+
+  var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
+  if (origins.indexOf(origin) < 0)
+    return 'DIRECT';
+  if (isHttp)
+    return 'PROXY %(remote)s:%(httpport)s';
+  if (isHttps || isWebSocket || isWebSocketSSL)
+    return 'PROXY %(remote)s:%(sslport)s';
+  return 'DIRECT';
+}""" % { "origins": origins,
+         "remote":  webServer,
+         "httpport":httpPort,
+         "sslport": sslPort }
+        pacURL = "".join(pacURL.splitlines())
+
+        prefs.append(("network.proxy.type", 2))
+        prefs.append(("network.proxy.autoconfig_url", pacURL))
+
+        return prefs
+
+    def clean_permissions(self):
+        """Removed permissions added by mozprofile."""
+
+        # Open database and create table
+        permDB = sqlite3.connect(os.path.join(self._profileDir, "permissions.sqlite"))
+        cursor = permDB.cursor();
+
+        # TODO: only delete values that we add, this would require sending in the full permissions object
+        cursor.execute("DROP TABLE IF EXISTS moz_hosts");
+
+        # Commit and close
+        permDB.commit()
+        cursor.close()

testing/mozbase/mozprofile/mozprofile/prefs.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprofile.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2008
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Clint Talbert <ctalbert@mozilla.com>
+#   Jeff Hammel <jhammel@mozilla.com>
+#   Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""
+user preferences
+"""
+
+import os
+import re
+from ConfigParser import SafeConfigParser as ConfigParser
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+class PreferencesReadError(Exception):
+    """read error for prefrences files"""
+
+
+class Preferences(object):
+    """assembly of preferences from various sources"""
+
+    def __init__(self, prefs=None):
+        self._prefs = []
+        if prefs:
+            self.add(prefs)
+
+    def add(self, prefs, cast=False):
+        """
+        - cast: whether to cast strings to value, e.g. '1' -> 1
+        """
+        # wants a list of 2-tuples
+        if isinstance(prefs, dict):
+            prefs = prefs.items()
+        if cast:
+            prefs = [(i, self.cast(j)) for i, j in prefs]
+        self._prefs += prefs
+
+    def add_file(self, path):
+        """a preferences from a file"""
+        self.add(self.read(path))
+
+    def __call__(self):
+        return self._prefs
+
+    @classmethod
+    def cast(cls, value):
+        """
+        interpolate a preference from a string
+        from the command line or from e.g. an .ini file, there is no good way to denote
+        what type the preference value is, as natively it is a string
+        - integers will get cast to integers
+        - true/false will get cast to True/False
+        - anything enclosed in single quotes will be treated as a string with the ''s removed from both sides
+        """
+
+        if not isinstance(value, basestring):
+            return value # no op
+        quote = "'"
+        if value == 'true':
+            return  True
+        if value == 'false':
+            return False
+        try:
+            return int(value)
+        except ValueError:
+            pass
+        if value.startswith(quote) and value.endswith(quote):
+            value = value[1:-1]
+        return value
+
+
+    @classmethod
+    def read(cls, path):
+        """read preferences from a file"""
+
+        section = None # for .ini files
+        basename = os.path.basename(path)
+        if ':' in basename:
+            # section of INI file
+            path, section = path.rsplit(':', 1)
+
+        if not os.path.exists(path):
+            raise PreferencesReadError("'%s' does not exist" % path)
+
+        if section:
+            try:
+                return cls.read_ini(path, section)
+            except PreferencesReadError:
+                raise
+            except Exception, e:
+                raise PreferencesReadError(str(e))
+
+        # try both JSON and .ini format
+        try:
+            return cls.read_json(path)
+        except Exception, e:
+            try:
+                return cls.read_ini(path)
+            except Exception, f:
+                for exception in e, f:
+                    if isinstance(exception, PreferencesReadError):
+                        raise exception
+                raise PreferencesReadError("Could not recognize format of %s" % path)
+
+
+    @classmethod
+    def read_ini(cls, path, section=None):
+        """read preferences from an .ini file"""
+
+        parser = ConfigParser()
+        parser.read(path)
+
+        if section:
+            if section not in parser.sections():
+                raise PreferencesReadError("No section '%s' in %s" % (section, path))
+            retval = parser.items(section, raw=True)
+        else:
+            retval = parser.defaults().items()
+
+        # cast the preferences since .ini is just strings
+        return [(i, cls.cast(j)) for i, j in retval]
+
+    @classmethod
+    def read_json(cls, path):
+        """read preferences from a JSON blob"""
+
+        prefs = json.loads(file(path).read())
+
+        if type(prefs) not in [list, dict]:
+            raise PreferencesReadError("Malformed preferences: %s" % path)
+        if isinstance(prefs, list):
+            if [i for i in prefs if type(i) != list or len(i) != 2]:
+                raise PreferencesReadError("Malformed preferences: %s" % path)
+            values = [i[1] for i in prefs]
+        elif isinstance(prefs, dict):
+            values = prefs.values()
+        else:
+            raise PreferencesReadError("Malformed preferences: %s" % path)
+        types = (bool, basestring, int)
+        if [i for i in values
+            if not [isinstance(i, j) for j in types]]:
+            raise PreferencesReadError("Only bool, string, and int values allowed")
+        return prefs
+
+    @classmethod
+    def read_prefs(cls, path, pref_setter='user_pref'):
+        """read preferences from (e.g.) prefs.js"""
+
+        comment = re.compile('/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/', re.MULTILINE)
+
+        token = '##//' # magical token
+        lines = [i.strip() for i in file(path).readlines() if i.strip()]
+        _lines = []
+        for line in lines:
+            if line.startswith('#'):
+                continue
+            if '//' in line:
+                line = line.replace('//', token)
+            _lines.append(line)
+        string = '\n'.join(_lines)
+        string = re.sub(comment, '', string)
+
+        retval = []
+        def pref(a, b):
+            retval.append((a, b))
+        lines = [i.strip().rstrip(';') for i in string.split('\n') if i.strip()]
+
+        _globals = {'retval': retval, 'true': True, 'false': False}
+        _globals[pref_setter] = pref
+        for line in lines:
+            try:
+                eval(line, _globals, {})
+            except SyntaxError:
+                print line
+                raise
+
+        # de-magic the token
+        for index, (key, value) in enumerate(retval):
+            if isinstance(value, basestring) and token in value:
+                retval[index] = (key, value.replace(token, '//'))
+
+        return retval
+
+    @classmethod
+    def write(_file, prefs, pref_string='user_pref("%s", %s);'):
+        """write preferences to a file"""
+
+        if isinstance(_file, basestring):
+            f = file(_file, 'w')
+        else:
+            f = _file
+
+        if isinstance(prefs, dict):
+            prefs = prefs.items()
+
+        for key, value in prefs:
+            if value is True:
+                print >> f, pref_string % (key, 'true')
+            elif value is False:
+                print >> f, pref_string % (key, 'false')
+            elif isinstance(value, basestring):
+                print >> f, pref_string % (key, repr(string(value)))
+            else:
+                print >> f, pref_string % (key, value) # should be numeric!
+
+        if isinstance(_file, basestring):
+            f.close()
+
+if __name__ == '__main__':
+    pass

testing/mozbase/mozprofile/mozprofile/profile.py

@@ -0,0 +1,272 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprofile.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2008-2009
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mikeal Rogers <mikeal.rogers@gmail.com>
+#   Clint Talbert <ctalbert@mozilla.com>
+#   Henrik Skupin <hskupin@mozilla.com>
+#   Jeff Hammel <jhammel@mozilla.com>
+#   Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+__all__ = ['Profile', 'FirefoxProfile', 'ThunderbirdProfile']
+
+import os
+import tempfile
+from addons import AddonManager
+from permissions import PermissionsManager
+from shutil import rmtree
+
+try:
+    import simplejson
+except ImportError:
+    import json as simplejson
+
+class Profile(object):
+    """Handles all operations regarding profile. Created new profiles, installs extensions,
+    sets preferences and handles cleanup."""
+
+    def __init__(self, profile=None, addons=None, addon_manifests=None, preferences=None, locations=None, proxy=False, restore=True):
+
+        # if true, remove installed addons/prefs afterwards
+        self.restore = restore
+
+        # Handle profile creation
+        self.create_new = not profile
+        if profile:
+            # Ensure we have a full path to the profile
+            self.profile = os.path.abspath(os.path.expanduser(profile))
+            if not os.path.exists(self.profile):
+                os.makedirs(self.profile)
+        else:
+            self.profile = self.create_new_profile()
+
+        # set preferences
+        if hasattr(self.__class__, 'preferences'):
+            # class preferences
+            self.set_preferences(self.__class__.preferences)
+        self._preferences = preferences
+        if preferences:
+            # supplied preferences
+            if isinstance(preferences, dict):
+                # unordered
+                preferences = preferences.items()
+            # sanity check
+            assert not [i for i in preferences
+                        if len(i) != 2]
+        else:
+            preferences = []
+        self.set_preferences(preferences)
+
+        # set permissions
+        self._locations = locations # store this for reconstruction
+        self._proxy = proxy
+        self.permission_manager = PermissionsManager(self.profile, locations)
+        prefs_js, user_js = self.permission_manager.getNetworkPreferences(proxy)
+        self.set_preferences(prefs_js, 'prefs.js')
+        self.set_preferences(user_js)
+
+        # handle addon installation
+        self.addon_manager = AddonManager(self.profile)
+        self.addon_manager.install_addons(addons, addon_manifests)
+
+    def exists(self):
+        """returns whether the profile exists or not"""
+        return os.path.exists(self.profile)
+
+    def reset(self):
+        """
+        reset the profile to the beginning state
+        """
+        self.cleanup()
+        if self.create_new:
+            profile = None
+        else:
+            profile = self.profile
+        self.__init__(profile=profile,
+                      addons=self.addon_manager.addons,
+                      addon_manifests=self.addon_manager.manifests,
+                      preferences=self._preferences,
+                      locations=self._locations,
+                      proxy = self._proxy)
+
+    def create_new_profile(self):
+        """Create a new clean profile in tmp which is a simple empty folder"""
+        profile = tempfile.mkdtemp(suffix='.mozrunner')
+        return profile
+
+
+    ### methods for preferences
+
+    def set_preferences(self, preferences, filename='user.js'):
+        """Adds preferences dict to profile preferences"""
+
+        # append to the file
+        prefs_file = os.path.join(self.profile, filename)
+        f = open(prefs_file, 'a')
+
+        if isinstance(preferences, dict):
+            # order doesn't matter
+            preferences = preferences.items()
+
+        # write the preferences
+        if preferences:
+            f.write('\n#MozRunner Prefs Start\n')
+            _prefs = [(simplejson.dumps(k), simplejson.dumps(v) )
+                      for k, v in preferences]
+            for _pref in _prefs:
+                f.write('user_pref(%s, %s);\n' % _pref)
+            f.write('#MozRunner Prefs End\n')
+        f.close()
+
+    def pop_preferences(self):
+        """
+        pop the last set of preferences added
+        returns True if popped
+        """
+
+        # our magic markers
+        delimeters = ('#MozRunner Prefs Start', '#MozRunner Prefs End')
+
+        lines = file(os.path.join(self.profile, 'user.js')).read().splitlines()
+        def last_index(_list, value):
+            """
+            returns the last index of an item;
+            this should actually be part of python code but it isn't
+            """
+            for index in reversed(range(len(_list))):
+                if _list[index] == value:
+                    return index
+        s = last_index(lines, delimeters[0])
+        e = last_index(lines, delimeters[1])
+
+        # ensure both markers are found
+        if s is None:
+            assert e is None, '%s found without %s' % (delimeters[1], delimeters[0])
+            return False # no preferences found
+        elif e is None:
+            assert e is None, '%s found without %s' % (delimeters[0], delimeters[1])
+
+        # ensure the markers are in the proper order
+        assert e > s, '%s found at %s, while %s found at %s' (delimeter[1], e, delimeter[0], s)
+
+        # write the prefs
+        cleaned_prefs = '\n'.join(lines[:s] + lines[e+1:])
+        f = file(os.path.join(self.profile, 'user.js'), 'w')
+        return True
+
+    def clean_preferences(self):
+        """Removed preferences added by mozrunner."""
+        while True:
+            if not self.pop_preferences():
+                break
+
+    ### cleanup
+
+    def _cleanup_error(self, function, path, excinfo):
+        """ Specifically for windows we need to handle the case where the windows
+            process has not yet relinquished handles on files, so we do a wait/try
+            construct and timeout if we can't get a clear road to deletion
+        """
+        try:
+            from exceptions import WindowsError
+            from time import sleep
+            def is_file_locked():
+                return excinfo[0] is WindowsError and excinfo[1].winerror == 32
+
+            if excinfo[0] is WindowsError and excinfo[1].winerror == 32:
+                # Then we're on windows, wait to see if the file gets unlocked
+                # we wait 10s
+                count = 0
+                while count < 10:
+                    sleep(1)
+                    try:
+                        function(path)
+                        break
+                    except:
+                        count += 1
+        except ImportError:
+            # We can't re-raise an error, so we'll hope the stuff above us will throw
+            pass
+
+
+    def cleanup(self):
+        """Cleanup operations on the profile."""
+        if self.restore:
+            if self.create_new:
+                if os.path.exists(self.profile):
+                    rmtree(self.profile, onerror=self._cleanup_error)
+            else:
+                self.clean_preferences()
+                self.addon_manager.clean_addons()
+                self.permission_manager.clean_permissions()
+
+    __del__ = cleanup
+
+class FirefoxProfile(Profile):
+    """Specialized Profile subclass for Firefox"""
+    preferences = {# Don't automatically update the application
+                   'app.update.enabled' : False,
+                   # Don't restore the last open set of tabs if the browser has crashed
+                   'browser.sessionstore.resume_from_crash': False,
+                   # Don't check for the default web browser
+                   'browser.shell.checkDefaultBrowser' : False,
+                   # Don't warn on exit when multiple tabs are open
+                   'browser.tabs.warnOnClose' : False,
+                   # Don't warn when exiting the browser
+                   'browser.warnOnQuit': False,
+                   # Only install add-ons from the profile and the application scope
+                   # Also ensure that those are not getting disabled.
+                   # see: https://developer.mozilla.org/en/Installing_extensions
+                   'extensions.enabledScopes' : 5,
+                   'extensions.autoDisableScopes' : 10,
+                   # Don't install distribution add-ons from the app folder
+                   'extensions.installDistroAddons' : False,
+                   # Dont' run the add-on compatibility check during start-up
+                   'extensions.showMismatchUI' : False,
+                   # Don't automatically update add-ons
+                   'extensions.update.enabled'    : False,
+                   # Don't open a dialog to show available add-on updates
+                   'extensions.update.notifyUser' : False,
+                   }
+
+class ThunderbirdProfile(Profile):
+    preferences = {'extensions.update.enabled'    : False,
+                   'extensions.update.notifyUser' : False,
+                   'browser.shell.checkDefaultBrowser' : False,
+                   'browser.tabs.warnOnClose' : False,
+                   'browser.warnOnQuit': False,
+                   'browser.sessionstore.resume_from_crash': False,
+                   # prevents the 'new e-mail address' wizard on new profile
+                   'mail.provider.enabled': False,
+                   }

testing/mozbase/mozprofile/setup.py

@@ -0,0 +1,83 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozprofile.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2008
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mikeal Rogers <mikeal.rogers@gmail.com>
+#   Clint Talbert <ctalbert@mozilla.com>
+#   Jeff Hammel <jhammel@mozilla.com>
+#   Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os
+import sys
+from setuptools import setup, find_packages
+
+version = '0.1b2'
+
+# we only support python 2 right now
+assert sys.version_info[0] == 2
+
+deps = ["ManifestDestiny == 0.5.4"]
+# version-dependent dependencies
+try:
+    import json
+except ImportError:
+    deps.append('simplejson')
+
+# take description from README
+here = os.path.dirname(os.path.abspath(__file__))
+try:
+    description = file(os.path.join(here, 'README.md')).read()
+except (OSError, IOError):
+    description = ''
+
+setup(name='mozprofile',
+      version=version,
+      description="handling of Mozilla XUL app profiles",
+      long_description=description,
+      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+      keywords='',
+      author='Mozilla Automation + Testing Team',
+      author_email='mozmill-dev@googlegroups.com',
+      url='http://github.com/mozautomation/mozmill',
+      license='MPL',
+      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
+      include_package_data=True,
+      zip_safe=False,
+      install_requires=deps,
+      entry_points="""
+      # -*- Entry points: -*-
+      [console_scripts]
+      mozprofile = mozprofile:cli
+      """,
+      )

testing/mozbase/mozrunner/README.md

@@ -0,0 +1,43 @@
+[mozrunner](https://github.com/mozilla/mozbase/tree/master/mozrunner)
+is a [python package](http://pypi.python.org/pypi/mozrunner)
+which handles running of Mozilla applications.
+mozrunner utilizes [mozprofile](/en/Mozprofile)
+for managing application profiles
+and [mozprocess](/en/Mozprocess) for robust process control.
+
+mozrunner may be used from the command line or programmatically as an API.
+
+
+# Command Line Usage
+
+The `mozrunner` command will launch the application (specified by
+`--app`) from a binary specified with `-b` or as located on the `PATH`.
+
+mozrunner takes the command line options from 
+[mozprofile](/en/Mozprofile) for constructing the profile to be used by 
+the application.
+
+Run `mozrunner --help` for detailed information on the command line
+program.
+
+
+# API Usage
+
+mozrunner features a base class, 
+[mozrunner.runner.Runner](https://github.com/mozilla/mozbase/blob/master/mozrunner/mozrunner/runner.py) 
+which is an integration layer API for interfacing with Mozilla applications.
+
+mozrunner also exposes two application specific classes,
+`FirefoxRunner` and `ThunderbirdRunner` which record the binary names
+necessary for the `Runner` class to find them on the system.
+
+Example API usage:
+
+    from mozrunner import FirefoxRunner
+	
+    # start Firefox on a new profile
+    runner = FirefoxRunner()
+    runner.start()
+
+See also a comparable implementation for [selenium](http://seleniumhq.org/): 
+http://code.google.com/p/selenium/source/browse/trunk/py/selenium/webdriver/firefox/firefox_binary.py

testing/mozbase/mozrunner/mozrunner/__init__.py

@@ -0,0 +1,40 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozrunner.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2008-2009
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Mikeal Rogers <mikeal.rogers@gmail.com>
+#  Clint Talbert <ctalbert@mozilla.com>
+#  Henrik Skupin <hskupin@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+from runner import *

testing/mozbase/mozrunner/mozrunner/runner.py

@@ -0,0 +1,436 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozrunner.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2008-2009
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mikeal Rogers <mikeal.rogers@gmail.com>
+#   Clint Talbert <ctalbert@mozilla.com>
+#   Henrik Skupin <hskupin@mozilla.com>
+#   Jeff Hammel <jhammel@mozilla.com>
+#   Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+__all__ = ['Runner', 'ThunderbirdRunner', 'FirefoxRunner', 'runners', 'CLI', 'cli', 'package_metadata']
+
+import mozinfo
+import optparse
+import os
+import sys
+import ConfigParser
+
+from utils import get_metadata_from_egg
+from utils import findInPath
+from mozprofile import *
+from mozprocess.processhandler import ProcessHandler
+
+package_metadata = get_metadata_from_egg('mozrunner')
+
+class BinaryLocationException(Exception):
+    """exception for failure to find the binary"""
+
+
+class Runner(object):
+    """Handles all running operations. Finds bins, runs and kills the process."""
+
+    ### data to be filled in by subclasses
+    profile = Profile # profile class to use by default
+    names = [] # names of application to look for on PATH
+    app_name = '' # name of application in windows registry
+    program_names = [] # names of application in windows program files
+
+    @classmethod
+    def create(cls, binary=None, cmdargs=None, env=None, kp_kwargs=None, profile_args=None,
+                                               clean_profile=True, process_class=ProcessHandler):
+        profile = cls.profile_class(**(profile_args or {}))
+        return cls(profile, binary=binary, cmdargs=cmdargs, env=env, kp_kwargs=kp_kwargs,
+                                           clean_profile=clean_profile, process_class=process_class)
+
+    def __init__(self, profile, binary=None, cmdargs=None, env=None,
+                 kp_kwargs=None, clean_profile=True, process_class=ProcessHandler):
+        self.process_handler = None
+        self.process_class = process_class
+        self.profile = profile
+        self.clean_profile = clean_profile
+
+        self.firstrun = False
+
+        # find the binary
+        self.binary = self.__class__.get_binary(binary)
+        if not os.path.exists(self.binary):
+            raise OSError("Binary path does not exist: %s" % self.binary)
+
+        self.cmdargs = cmdargs or []
+        _cmdargs = [i for i in self.cmdargs
+                    if i != '-foreground']
+        if len(_cmdargs) != len(self.cmdargs):
+            # foreground should be last; see
+            # - https://bugzilla.mozilla.org/show_bug.cgi?id=625614
+            # - https://bugzilla.mozilla.org/show_bug.cgi?id=626826
+            self.cmdargs = _cmdargs
+            self.cmdargs.append('-foreground')
+
+        # process environment
+        if env is None:
+            self.env = os.environ.copy()
+        else:
+            self.env = env.copy()
+        # allows you to run an instance of Firefox separately from any other instances
+        self.env['MOZ_NO_REMOTE'] = '1'
+        # keeps Firefox attached to the terminal window after it starts
+        self.env['NO_EM_RESTART'] = '1'
+
+        # set the library path if needed on linux
+        if sys.platform == 'linux2' and self.binary.endswith('-bin'):
+            dirname = os.path.dirname(self.binary)
+            if os.environ.get('LD_LIBRARY_PATH', None):
+                self.env['LD_LIBRARY_PATH'] = '%s:%s' % (os.environ['LD_LIBRARY_PATH'], dirname)
+            else:
+                self.env['LD_LIBRARY_PATH'] = dirname
+
+        # arguments for ProfessHandler.Process
+        self.kp_kwargs = kp_kwargs or {}
+
+    @classmethod
+    def get_binary(cls, binary=None):
+        """determine the binary"""
+        if binary is None:
+            binary = cls.find_binary()
+            if binary is None:
+                raise BinaryLocationException("Your binary could not be located; you will need to set it")
+            return binary
+        elif mozinfo.isMac and binary.find('Contents/MacOS/') == -1:
+            return os.path.join(binary, 'Contents/MacOS/%s-bin' % cls.names[0])
+        else:
+            return binary
+
+    @classmethod
+    def find_binary(cls):
+        """Finds the binary for class names if one was not provided."""
+
+        binary = None
+        if mozinfo.isUnix:
+            for name in cls.names:
+                binary = findInPath(name)
+                if binary:
+                    return binary
+        elif mozinfo.isWin:
+
+            # find the default executable from the windows registry
+            try:
+                # assumes cls.app_name is defined, as it should be for implementors
+                import _winreg
+                app_key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r"Software\Mozilla\Mozilla %s" % cls.app_name)
+                version, _type = _winreg.QueryValueEx(app_key, "CurrentVersion")
+                version_key = _winreg.OpenKey(app_key, version + r"\Main")
+                path, _ = _winreg.QueryValueEx(version_key, "PathToExe")
+                return path
+            except: # XXX not sure what type of exception this should be
+                pass
+
+            # search for the binary in the path
+            for name in cls.names:
+                binary = findInPath(name)
+                if binary:
+                    return binary
+
+            # search for the binary in program files
+            if sys.platform == 'cygwin':
+                program_files = os.environ['PROGRAMFILES']
+            else:
+                program_files = os.environ['ProgramFiles']
+
+            program_files = [program_files]
+            if  "ProgramFiles(x86)" in os.environ:
+                program_files.append(os.environ["ProgramFiles(x86)"])
+            for program_file in program_files:
+                for program_name in cls.program_names:
+                    path = os.path.join(program_name, program_file, 'firefox.exe')
+                    if os.path.isfile(path):
+                        return path
+
+        elif mozinfo.isMac:
+            for name in cls.names:
+                appdir = os.path.join('Applications', name.capitalize()+'.app')
+                if os.path.isdir(os.path.join(os.path.expanduser('~/'), appdir)):
+                    binary = os.path.join(os.path.expanduser('~/'), appdir,
+                                          'Contents/MacOS/'+name+'-bin')
+                elif os.path.isdir('/'+appdir):
+                    binary = os.path.join("/"+appdir, 'Contents/MacOS/'+name+'-bin')
+
+                if binary is not None:
+                    if not os.path.isfile(binary):
+                        binary = binary.replace(name+'-bin', 'firefox-bin')
+                    if not os.path.isfile(binary):
+                        binary = None
+                if binary:
+                    return binary
+        return binary
+
+    @property
+    def command(self):
+        """Returns the command list to run."""
+        return [self.binary, '-profile', self.profile.profile]
+
+    def get_repositoryInfo(self):
+        """Read repository information from application.ini and platform.ini."""
+
+        config = ConfigParser.RawConfigParser()
+        dirname = os.path.dirname(self.binary)
+        repository = { }
+
+        for file, section in [('application', 'App'), ('platform', 'Build')]:
+            config.read(os.path.join(dirname, '%s.ini' % file))
+
+            for key, id in [('SourceRepository', 'repository'),
+                            ('SourceStamp', 'changeset')]:
+                try:
+                    repository['%s_%s' % (file, id)] = config.get(section, key);
+                except:
+                    repository['%s_%s' % (file, id)] = None
+
+        return repository
+
+    def is_running(self):
+        return self.process_handler is not None
+
+    def start(self):
+        """Run self.command in the proper environment."""
+
+        # ensure you are stopped
+        self.stop()
+
+        # ensure the profile exists
+        if not self.profile.exists():
+            self.profile.reset()
+            self.firstrun = False
+
+        # run once to register any extensions
+        # see:
+        # - http://hg.mozilla.org/releases/mozilla-1.9.2/file/915a35e15cde/build/automation.py.in#l702
+        # - http://mozilla-xp.com/mozilla.dev.apps.firefox/Rules-for-when-firefox-bin-restarts-it-s-process
+        # This run just calls through processhandler to popen directly as we
+        # are not particuarly cared in tracking this process
+        if not self.firstrun:
+            firstrun = ProcessHandler.Process(self.command+['-silent', '-foreground'], env=self.env, **self.kp_kwargs)
+            firstrun.wait()
+            self.firstrun = True
+
+        # now run for real, this run uses the managed processhandler
+        self.process_handler = self.process_class(self.command+self.cmdargs, env=self.env, **self.kp_kwargs)
+        self.process_handler.run()
+
+    def wait(self, timeout=None, outputTimeout=None):
+        """Wait for the app to exit."""
+        if self.process_handler is None:
+            return
+        self.process_handler.waitForFinish(timeout=timeout, outputTimeout=outputTimeout)
+        self.process_handler = None
+
+    def stop(self):
+        """Kill the app"""
+        if self.process_handler is None:
+            return
+        self.process_handler.kill()
+        self.process_handler = None
+
+    def reset(self):
+        """
+        reset the runner between runs
+        currently, only resets the profile, but probably should do more
+        """
+        self.profile.reset()
+
+    def cleanup(self):
+        self.stop()
+        if self.clean_profile:
+            self.profile.cleanup()
+
+    __del__ = cleanup
+
+
+class FirefoxRunner(Runner):
+    """Specialized Runner subclass for running Firefox."""
+
+    app_name = 'Firefox'
+    profile_class = FirefoxProfile
+    program_names = ['Mozilla Firefox']
+
+    # (platform-dependent) names of binary
+    if mozinfo.isMac:
+        names = ['firefox', 'minefield', 'shiretoko']
+    elif mozinfo.isUnix:
+        names = ['firefox', 'mozilla-firefox', 'iceweasel']
+    elif mozinfo.isWin:
+        names =['firefox']
+    else:
+        raise AssertionError("I don't know what platform you're on")
+
+    def __init__(self, profile, **kwargs):
+        Runner.__init__(self, profile, **kwargs)
+
+        # Find application version number
+        appdir = os.path.dirname(os.path.realpath(self.binary))
+        appini = ConfigParser.RawConfigParser()
+        appini.read(os.path.join(appdir, 'application.ini'))
+        # Version needs to be of the form 3.6 or 4.0b and not the whole string
+        version = appini.get('App', 'Version').rstrip('0123456789pre').rstrip('.')
+
+        # Disable compatibility check. See:
+        # - http://kb.mozillazine.org/Extensions.checkCompatibility
+        # - https://bugzilla.mozilla.org/show_bug.cgi?id=659048
+        preference = {'extensions.checkCompatibility.' + version: False,
+                      'extensions.checkCompatibility.nightly': False}
+        self.profile.set_preferences(preference)
+
+    @classmethod
+    def get_binary(cls, binary=None):
+        if (not binary) and 'BROWSER_PATH' in os.environ:
+            return os.environ['BROWSER_PATH']
+        return Runner.get_binary(binary)
+
+class ThunderbirdRunner(Runner):
+    """Specialized Runner subclass for running Thunderbird"""
+    app_name = 'Thunderbird'
+    profile_class = ThunderbirdProfile
+    names = ["thunderbird", "shredder"]
+
+runners = {'firefox': FirefoxRunner,
+           'thunderbird': ThunderbirdRunner}
+
+class CLI(MozProfileCLI):
+    """Command line interface."""
+
+    module = "mozrunner"
+
+    def __init__(self, args=sys.argv[1:]):
+        """
+        Setup command line parser and parse arguments
+        - args : command line arguments
+        """
+
+        self.metadata = getattr(sys.modules[self.module],
+                                'package_metadata',
+                                {})
+        version = self.metadata.get('Version')
+        parser_args = {'description': self.metadata.get('Summary')}
+        if version:
+            parser_args['version'] = "%prog " + version
+        self.parser = optparse.OptionParser(**parser_args)
+        self.add_options(self.parser)
+        (self.options, self.args) = self.parser.parse_args(args)
+
+        if getattr(self.options, 'info', None):
+            self.print_metadata()
+            sys.exit(0)
+
+        # choose appropriate runner and profile classes
+        try:
+            self.runner_class = runners[self.options.app]
+        except KeyError:
+            self.parser.error('Application "%s" unknown (should be one of "firefox" or "thunderbird")' % self.options.app)
+
+    def add_options(self, parser):
+        """add options to the parser"""
+
+        # add profile options
+        MozProfileCLI.add_options(self, parser)
+
+        # add runner options
+        parser.add_option('-b', "--binary",
+                          dest="binary", help="Binary path.",
+                          metavar=None, default=None)
+        parser.add_option('--app', dest='app', default='firefox',
+                          help="Application to use [DEFAULT: %default]")
+        parser.add_option('--app-arg', dest='appArgs',
+                          default=[], action='append',
+                          help="provides an argument to the test application")
+        if self.metadata:
+            parser.add_option("--info", dest="info", default=False,
+                              action="store_true",
+                              help="Print module information")
+
+    ### methods for introspecting data
+
+    def get_metadata_from_egg(self):
+        import pkg_resources
+        ret = {}
+        dist = pkg_resources.get_distribution(self.module)
+        if dist.has_metadata("PKG-INFO"):
+            for line in dist.get_metadata_lines("PKG-INFO"):
+                key, value = line.split(':', 1)
+                ret[key] = value
+        if dist.has_metadata("requires.txt"):
+            ret["Dependencies"] = "\n" + dist.get_metadata("requires.txt")
+        return ret
+
+    def print_metadata(self, data=("Name", "Version", "Summary", "Home-page",
+                                   "Author", "Author-email", "License", "Platform", "Dependencies")):
+        for key in data:
+            if key in self.metadata:
+                print key + ": " + self.metadata[key]
+
+    ### methods for running
+
+    def command_args(self):
+        """additional arguments for the mozilla application"""
+        return self.options.appArgs
+
+    def runner_args(self):
+        """arguments to instantiate the runner class"""
+        return dict(cmdargs=self.command_args(),
+                    binary=self.options.binary,
+                    profile_args=self.profile_args())
+
+    def create_runner(self):
+        return self.runner_class.create(**self.runner_args())
+
+    def run(self):
+        runner = self.create_runner()
+        self.start(runner)
+        runner.cleanup()
+
+    def start(self, runner):
+        """Starts the runner and waits for Firefox to exit or Keyboard Interrupt.
+        Shoule be overwritten to provide custom running of the runner instance."""
+        runner.start()
+        print 'Starting:', ' '.join(runner.command)
+        try:
+            runner.wait()
+        except KeyboardInterrupt:
+            runner.stop()
+
+
+def cli(args=sys.argv[1:]):
+    CLI(args).run()
+
+if __name__ == '__main__':
+    cli()

testing/mozbase/mozrunner/mozrunner/utils.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozrunner.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2008-2009
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mikeal Rogers <mikeal.rogers@gmail.com>
+#   Clint Talbert <ctalbert@mozilla.com>
+#   Henrik Skupin <hskupin@mozilla.com>
+#   Jeff Hammel <jhammel@mozilla.com>
+#   Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""
+utility functions for mozrunner
+"""
+
+__all__ = ['findInPath', 'get_metadata_from_egg']
+
+import mozinfo
+import os
+import sys
+
+
+### python package method metadata by introspection
+try:
+    import pkg_resources
+    def get_metadata_from_egg(module):
+        ret = {}
+        dist = pkg_resources.get_distribution(module)
+        if dist.has_metadata("PKG-INFO"):
+            key = None
+            for line in dist.get_metadata("PKG-INFO").splitlines():
+                # see http://www.python.org/dev/peps/pep-0314/
+                if key == 'Description':
+                    # descriptions can be long
+                    if not line or line[0].isspace():
+                        value += '\n' + line
+                        continue
+                    else:
+                        key = key.strip()
+                        value = value.strip()
+                        ret[key] = value
+
+                key, value = line.split(':', 1)
+                key = key.strip()
+                value = value.strip()
+                ret[key] = value
+        if dist.has_metadata("requires.txt"):
+            ret["Dependencies"] = "\n" + dist.get_metadata("requires.txt")
+        return ret
+except ImportError:
+    # package resources not avaialable
+    def get_metadata_from_egg(module):
+        return {}
+
+
+def findInPath(fileName, path=os.environ['PATH']):
+    """python equivalent of which; should really be in the stdlib"""
+    dirs = path.split(os.pathsep)
+    for dir in dirs:
+        if os.path.isfile(os.path.join(dir, fileName)):
+            return os.path.join(dir, fileName)
+        if mozinfo.isWin:
+            if os.path.isfile(os.path.join(dir, fileName + ".exe")):
+                return os.path.join(dir, fileName + ".exe")
+
+if __name__ == '__main__':
+    for i in sys.argv[1:]:
+        print findInPath(i)

testing/mozbase/mozrunner/setup.py

@@ -0,0 +1,84 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozrunner.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2008
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mikeal Rogers <mikeal.rogers@gmail.com>
+#   Clint Talbert <ctalbert@mozilla.com>
+#   Jeff Hammel <jhammel@mozilla.com>
+#   Andrew Halberstadt <halbersa@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os
+import sys
+from setuptools import setup, find_packages
+
+PACKAGE_NAME = "mozrunner"
+PACKAGE_VERSION = "4.0"
+
+desc = """Reliable start/stop/configuration of Mozilla Applications (Firefox, Thunderbird, etc.)"""
+# take description from README
+here = os.path.dirname(os.path.abspath(__file__))
+try:
+    description = file(os.path.join(here, 'README.md')).read()
+except (OSError, IOError):
+    description = ''
+
+deps = ['mozprocess', 'mozprofile', 'mozinfo']
+
+# we only support python 2 right now
+assert sys.version_info[0] == 2
+
+setup(name=PACKAGE_NAME,
+      version=PACKAGE_VERSION,
+      description=desc,
+      long_description=description,
+      author='Mikeal Rogers, Mozilla',
+      author_email='mikeal.rogers@gmail.com',
+      url='http://github.com/mozautomation/mozmill',
+      license='MPL 1.1/GPL 2.0/LGPL 2.1',
+      packages=find_packages(exclude=['legacy']),
+      zip_safe=False,
+      entry_points="""
+          [console_scripts]
+          mozrunner = mozrunner:cli
+        """,
+      platforms =['Any'],
+      install_requires = deps,
+      classifiers=['Development Status :: 4 - Beta',
+                   'Environment :: Console',
+                   'Intended Audience :: Developers',
+                   'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)',
+                   'Operating System :: OS Independent',
+                   'Topic :: Software Development :: Libraries :: Python Modules',
+                  ]
+     )

testing/mozbase/setup_development.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozbase.
+#
+# The Initial Developer of the Original Code is
+#   The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hammel <jhammel@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""
+Setup mozbase packages for development.
+
+Packages may be specified as command line arguments.
+If no arguments are given, install all packages.
+
+See https://wiki.mozilla.org/Auto-tools/Projects/MozBase
+"""
+
+# XXX note that currently directory names must equal package names
+
+import pkg_resources
+import os
+import sys
+from optparse import OptionParser
+
+from subprocess import PIPE
+try:
+    from subprocess import check_call as call
+except ImportError:
+    from subprocess import call
+
+
+# directory containing this file
+here = os.path.dirname(os.path.abspath(__file__))
+
+# all python packages
+all_packages = [i for i in os.listdir(here)
+                if os.path.exists(os.path.join(here, i, 'setup.py'))]
+
+def cycle_check(order, dependencies):
+    """ensure no cyclic dependencies"""
+    order_dict = dict([(j, i) for i, j in enumerate(order)])
+    for package, deps in dependencies.items():
+        index = order_dict[package]
+        for d in deps:
+            assert index > order_dict[d], "Cyclic dependencies detected"
+
+def dependencies(directory):
+    """
+    get the dependencies of a package directory containing a setup.py
+    returns the package name and the list of dependencies
+    """
+    assert os.path.exists(os.path.join(directory, 'setup.py'))
+
+    # setup the egg info
+    call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE)
+
+    # get the .egg-info directory
+    egg_info = [i for i in os.listdir(directory)
+                if i.endswith('.egg-info')]
+    assert len(egg_info) == 1, 'Expected one .egg-info directory in %s, got: %s' % (directory, egg_info)
+    egg_info = os.path.join(directory, egg_info[0])
+    assert os.path.isdir(egg_info), "%s is not a directory" % egg_info
+
+    # read the dependencies
+    requires = os.path.join(egg_info, 'requires.txt')
+    if os.path.exists(requires):
+        dependencies = [i.strip() for i in file(requires).readlines() if i.strip()]
+    else:
+        dependencies = []
+
+    # read the package information
+    pkg_info = os.path.join(egg_info, 'PKG-INFO')
+    info_dict = {}
+    for line in file(pkg_info).readlines():
+        if not line or line[0].isspace():
+            continue # XXX neglects description
+        assert ':' in line
+        key, value = [i.strip() for i in line.split(':', 1)]
+        info_dict[key] = value
+
+
+    # return the information
+    return info_dict['Name'], dependencies
+
+def sanitize_dependency(dep):
+    """
+    remove version numbers from deps
+    """
+    for joiner in ('==', '<=', '>='):
+        if joiner in dep:
+            dep = dep.split(joiner, 1)[0].strip()
+            return dep # XXX only one joiner allowed right now
+    return dep
+
+
+def unroll_dependencies(dependencies):
+    """
+    unroll a set of dependencies to a flat list
+
+    dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']),
+                    'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']),
+                    'packageC': set(['packageE']),
+                    'packageE': set(['packageF', 'packageG']),
+                    'packageF': set(['packageG']),
+                    'packageX': set(['packageA', 'packageG'])}
+    """
+
+    order = []
+
+    # flatten all
+    packages = set(dependencies.keys())
+    for deps in dependencies.values():
+        packages.update(deps)
+
+    while len(order) != len(packages):
+
+        for package in packages.difference(order):
+            if set(dependencies.get(package, set())).issubset(order):
+                order.append(package)
+                break
+        else:
+            raise AssertionError("Cyclic dependencies detected")
+
+    cycle_check(order, dependencies) # sanity check
+
+    return order
+
+
+def main(args=sys.argv[1:]):
+
+    # parse command line options
+    usage = '%prog [options] [package] [package] [...]'
+    parser = OptionParser(usage=usage, description=__doc__)
+    parser.add_option('-d', '--dependencies', dest='list_dependencies',
+                      action='store_true', default=False,
+                      help="list dependencies for the packages")
+    parser.add_option('--list', action='store_true', default=False,
+                      help="list what will be installed")
+    options, packages = parser.parse_args(args)
+
+    if not packages:
+        # install all packages
+        packages = sorted(all_packages)
+
+    # ensure specified packages are in the list
+    assert set(packages).issubset(all_packages), "Packages should be in %s (You gave: %s)" % (all_packages, packages)
+
+    if options.list_dependencies:
+        # list the package dependencies
+        for package in packages:
+            print '%s: %s' % dependencies(os.path.join(here, package))
+        parser.exit()
+
+    # gather dependencies
+    deps = {}
+    mapping = {} # mapping from subdir name to package name
+    # core dependencies
+    for package in packages:
+        key, value = dependencies(os.path.join(here, package))
+        deps[key] = [sanitize_dependency(dep) for dep in value]
+        mapping[package] = key
+    # indirect dependencies
+    flag = True
+    while flag:
+        flag = False
+        for value in deps.values():
+            for dep in value:
+                if dep in all_packages and dep not in deps:
+                    key, value = dependencies(os.path.join(here, dep))
+                    deps[key] = [sanitize_dependency(dep) for dep in value]
+                    mapping[package] = key
+                    flag = True
+                    break
+            if flag:
+                break
+
+    # get the remaining names for the mapping
+    for package in all_packages:
+        if package in mapping:
+            continue
+        key, value = dependencies(os.path.join(here, package))
+        mapping[package] = key
+
+    # unroll dependencies
+    unrolled = unroll_dependencies(deps)
+
+    # make a reverse mapping: package name -> subdirectory
+    reverse_mapping = dict([(j,i) for i, j in mapping.items()])
+
+    # we only care about dependencies in mozbase
+    unrolled = [package for package in unrolled if package in reverse_mapping]
+
+    if options.list:
+        # list what will be installed
+        for package in unrolled:
+            print package
+        parser.exit()
+
+    # set up the packages for development
+    for package in unrolled:
+        call([sys.executable, 'setup.py', 'develop'],
+             cwd=os.path.join(here, reverse_mapping[package]))
+
+if __name__ == '__main__':
+    main()

testing/testsuite-targets.mk

@@ -233,7 +233,7 @@ include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 ifndef UNIVERSAL_BINARY
 PKG_STAGE = $(DIST)/test-package-stage
-package-tests: stage-mochitest stage-reftest stage-xpcshell stage-jstests stage-jetpack stage-firebug stage-peptest
+package-tests: stage-mochitest stage-reftest stage-xpcshell stage-jstests stage-jetpack stage-firebug stage-peptest stage-mozbase
 else
 # This staging area has been built for us by universal/flight.mk
 PKG_STAGE = $(DIST)/universal/test-package-stage
@@ -255,7 +255,7 @@ package-tests: stage-android
 endif
 
 make-stage-dir:
-	rm -rf $(PKG_STAGE) && $(NSINSTALL) -D $(PKG_STAGE) && $(NSINSTALL) -D $(PKG_STAGE)/bin && $(NSINSTALL) -D $(PKG_STAGE)/bin/components && $(NSINSTALL) -D $(PKG_STAGE)/certs && $(NSINSTALL) -D $(PKG_STAGE)/jetpack && $(NSINSTALL) -D $(PKG_STAGE)/firebug && $(NSINSTALL) -D $(PKG_STAGE)/peptest
+	rm -rf $(PKG_STAGE) && $(NSINSTALL) -D $(PKG_STAGE) && $(NSINSTALL) -D $(PKG_STAGE)/bin && $(NSINSTALL) -D $(PKG_STAGE)/bin/components && $(NSINSTALL) -D $(PKG_STAGE)/certs && $(NSINSTALL) -D $(PKG_STAGE)/jetpack && $(NSINSTALL) -D $(PKG_STAGE)/firebug && $(NSINSTALL) -D $(PKG_STAGE)/peptest && $(NSINSTALL) -D $(PKG_STAGE)/mozbase
 
 stage-mochitest: make-stage-dir
 	$(MAKE) -C $(DEPTH)/testing/mochitest stage-package
@@ -283,9 +283,12 @@ stage-firebug: make-stage-dir
 
 stage-peptest: make-stage-dir
 	$(MAKE) -C $(DEPTH)/testing/peptest stage-package
+
+stage-mozbase: make-stage-dir
+	$(MAKE) -C $(DEPTH)/testing/mozbase stage-package
 .PHONY: \
   mochitest mochitest-plain mochitest-chrome mochitest-a11y mochitest-ipcplugins \
   reftest crashtest \
   xpcshell-tests \
   jstestbrowser \
-  package-tests make-stage-dir stage-mochitest stage-reftest stage-xpcshell stage-jstests stage-android stage-jetpack stage-firebug stage-peptest
+  package-tests make-stage-dir stage-mochitest stage-reftest stage-xpcshell stage-jstests stage-android stage-jetpack stage-firebug stage-peptest stage-mozbase

toolkit/toolkit-makefiles.sh

@@ -891,6 +891,7 @@ if [ "$ENABLE_TESTS" ]; then
     testing/xpcshell/example/Makefile
     testing/firebug/Makefile
     testing/peptest/Makefile
+    testing/mozbase/Makefile
     toolkit/components/alerts/test/Makefile
     toolkit/components/autocomplete/tests/Makefile
     toolkit/components/commandlines/test/Makefile

toolkit/toolkit-tiers.mk

@@ -268,5 +268,6 @@ tier_platform_dirs += testing/mochitest
 tier_platform_dirs += testing/xpcshell
 tier_platform_dirs += testing/tools/screenshot
 tier_platform_dirs += testing/peptest
+tier_platform_dirs += testing/mozbase
 endif