Bug 917999 - Part 1 - Write out and upload a manifest of test archives and the harnesses that depend on them.;r=ted

b2g/config/emulator-ics/config.json

@@ -10,6 +10,7 @@
         "{workdir}/out/target/product/generic/tests/*.zip",
         "{workdir}/out/emulator.tar.gz",
         "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
+        "{objdir}/dist/test_packages.json",
         "{workdir}/sources.xml"
     ],
     "upload_platform": "emulator-ics",

b2g/config/emulator/config.json

@@ -10,6 +10,7 @@
         "{workdir}/out/target/product/generic/tests/*.zip",
         "{workdir}/out/emulator.tar.gz",
         "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
+        "{objdir}/dist/test_packages.json",
         "{workdir}/sources.xml"
     ],
     "public_upload_files": [

build/gen_mach_buildprops.py

@@ -56,10 +56,12 @@ def getUrlProperties(filename):
         ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
         ('codeCoverageURL', lambda m: m.endswith('code-coverage-gcno.zip')),
         ('sdkUrl', lambda m: m.endswith(('sdk.tar.bz2', 'sdk.zip'))),
+        ('testPackagesUrl', lambda m: m.endswith('test_packages.json')),
         # packageUrl must be last!
-        ('packageUrl', lambda m: True),
+        ('packageUrl', lambda m: (not m.endswith('.json') and
+                                  not m.endswith('tests.zip'))),
     ]
-    url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
+    url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz|json))$''')
     properties = {}
 
     try:

build/gen_test_packages_manifest.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+
+from argparse import ArgumentParser
+
+ALL_HARNESSES = [
+    'common', # Harnesses without a specific package will look here.
+    'mochitest',
+    'reftest',
+    'webapprt',
+    'xpcshell',
+    'cppunittest',
+    'jittest',
+    'mozbase',
+]
+
+PACKAGE_SPECIFIED_HARNESSES = [
+]
+
+
+def parse_args():
+    parser = ArgumentParser(description='Generate a test_packages.json file to tell automation which harnesses require which test packages.')
+    parser.add_argument("--common", required=True,
+                        action="store", dest="tests_common",
+                        help="Name of the \"common\" archive, a package to be used by all harnesses.")
+    parser.add_argument("--jsshell", required=True,
+                        action="store", dest="jsshell",
+                        help="Name of the jsshell zip.")
+    for harness in PACKAGE_SPECIFIED_HARNESSES:
+        parser.add_argument("--%s" % harness, required=True,
+                            action="store", dest=harness,
+                            help="Name of the %s zip." % harness)
+    parser.add_argument("--dest-file", required=True,
+                        action="store", dest="destfile",
+                        help="Path to the output file to be written.")
+    return parser.parse_args()
+
+def generate_package_data(args):
+    # Generate a dictionary mapping test harness names (exactly as they're known to
+    # mozharness and testsuite-targets.mk, ideally) to the set of archive names that
+    # harness depends on to run.
+    # mozharness will use this file to determine what test zips to download,
+    # which will be an optimization once parts of the main zip are split to harness
+    # specific zips.
+    tests_common = args.tests_common
+    jsshell = args.jsshell
+
+    harness_requirements = dict([(k, [tests_common]) for k in ALL_HARNESSES])
+    harness_requirements['jittest'].append(jsshell)
+    for harness in PACKAGE_SPECIFIED_HARNESSES:
+        harness_requirements[harness].append(getattr(args, harness))
+    return harness_requirements
+
+if __name__ == '__main__':
+    args = parse_args()
+    packages_data = generate_package_data(args)
+    with open(args.destfile, 'w') as of:
+        json.dump(packages_data, of, indent=4)

testing/testsuite-targets.mk

@@ -406,6 +406,7 @@ package-tests: \
   stage-jittest \
   stage-web-platform-tests \
   stage-luciddream \
+  test-packages-manifest \
   $(NULL)
 ifdef MOZ_WEBRTC
 package-tests: stage-steeplechase
@@ -415,6 +416,10 @@ else
 PKG_STAGE = $(DIST)/universal/test-stage
 endif
 
+test-packages-manifest:
+	@rm -f $(MOZ_TEST_PACKAGES_FILE)
+	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py --common $(TEST_PACKAGE) --jsshell $(JSSHELL_NAME) --dest-file $(MOZ_TEST_PACKAGES_FILE)
+
 package-tests:
 	@rm -f '$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)'
 ifndef UNIVERSAL_BINARY
@@ -594,5 +599,6 @@ stage-instrumentation-tests: make-stage-dir
   stage-web-platform-tests \
   stage-instrumentation-tests \
   stage-luciddream \
+  test-packages-manifest \
   $(NULL)
 

toolkit/mozapps/installer/package-name.mk

@@ -155,8 +155,10 @@ endif
 MOZ_SOURCESTAMP_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).txt
 MOZ_BUILDINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).json
 MOZ_MOZINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).mozinfo.json
+MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/test_packages.json
 
 # JavaScript Shell
-PKG_JSSHELL = $(DIST)/jsshell-$(MOZ_PKG_PLATFORM).zip
+JSSHELL_NAME = jsshell-$(MOZ_PKG_PLATFORM).zip
+PKG_JSSHELL = $(DIST)/$(JSSHELL_NAME)
 
 endif # PACKAGE_NAME_MK_INCLUDED

toolkit/mozapps/installer/upload-files.mk

@@ -726,6 +726,7 @@ UPLOAD_FILES= \
   $(call QUOTED_WILDCARD,$(MOZ_SOURCESTAMP_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDINFO_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_MOZINFO_FILE)) \
+  $(call QUOTED_WILDCARD,$(MOZ_TEST_PACKAGES_FILE)) \
   $(call QUOTED_WILDCARD,$(PKG_JSSHELL)) \
   $(if $(UPLOAD_EXTRA_FILES), $(foreach f, $(UPLOAD_EXTRA_FILES), $(wildcard $(DIST)/$(f))))