Bug 1208320 - Produce xpcshell archive without staging test files; r=glandium

This commit produces the xpcshell test archive without staging 5000+
xpcshell test files first.

We teach the archiver to ignore .mkdir.done files.

The xpcshell Makefile.in still stages some files. This is less than
ideal. However, it is a small handful of files and shouldn't add too
much overhead.

This appears to not impact overall CPU usage significantly on my
machine, despute using Python instead of `zip`. It does reduce I/O
by ~25MB by avoiding the staging copy.

python/mozbuild/mozbuild/action/test_archive.py

@@ -16,9 +16,12 @@ import sys
 
 from mozpack.files import FileFinder
 from mozpack.mozjar import JarWriter
+import mozpack.path as mozpath
 
 import buildconfig
 
+STAGE = mozpath.join(buildconfig.topobjdir, 'dist', 'test-stage')
+
 
 ARCHIVE_FILES = {
     'mozharness': [
@@ -28,6 +31,19 @@ ARCHIVE_FILES = {
             'pattern': 'mozharness/**',
         },
     ],
+    'xpcshell': [
+        {
+            'source': buildconfig.topobjdir,
+            'base': '_tests/xpcshell',
+            'pattern': '**',
+            'dest': 'xpcshell/tests',
+        },
+        {
+            'source': STAGE,
+            'base': '',
+            'pattern': 'xpcshell/**',
+        },
+    ],
 }
 
 
@@ -36,13 +52,23 @@ def find_files(archive):
         source = entry['source']
         base = entry['base']
         pattern = entry['pattern']
+        dest = entry.get('dest')
+        ignore = list(entry.get('ignore', []))
+        ignore.append('**/.mkdir.done')
+        ignore.append('**/*.pyc')
 
-        finder = FileFinder(os.path.join(source, base),
-                            find_executables=False,
-                            find_dotfiles=True)
+        common_kwargs = {
+            'find_executables': False,
+            'find_dotfiles': True,
+            'ignore': ignore,
+        }
 
-        for f in finder.find(pattern):
-            yield f
+        finder = FileFinder(os.path.join(source, base), **common_kwargs)
+
+        for p, f in finder.find(pattern):
+            if dest:
+                p = mozpath.join(dest, p)
+            yield p, f
 
 
 def main(argv):
@@ -58,7 +84,8 @@ def main(argv):
 
     with open(args.outputfile, 'wb') as fh:
         with JarWriter(fileobj=fh, optimize=False) as writer:
-            for p, f in find_files(args.archive):
+            res = find_files(args.archive)
+            for p, f in res:
                 writer.add(p.encode('utf-8'), f.read(), mode=f.mode)
 
 

testing/testsuite-targets.mk

@@ -399,10 +399,15 @@ TEST_PKGS := \
   mochitest \
   reftest \
   talos \
-  xpcshell \
   web-platform \
   $(NULL)
 
+PYTHON_TEST_PKGS := \
+  xpcshell \
+  $(NULL)
+
+ALL_TEST_PKGS := $(TEST_PKGS) $(PYTHON_TEST_PKGS)
+
 PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.zip'
 
 test-packages-manifest-tc:
@@ -413,7 +418,7 @@ test-packages-manifest-tc:
       --dest-file $(MOZ_TEST_PACKAGES_FILE_TC) \
       --use-short-names \
       $(call PKG_ARG,common) \
-      $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
+      $(foreach pkg,$(ALL_TEST_PKGS),$(call PKG_ARG,$(pkg)))
 
 test-packages-manifest:
 	@rm -f $(MOZ_TEST_PACKAGES_FILE)
@@ -422,20 +427,22 @@ test-packages-manifest:
       --jsshell $(JSSHELL_NAME) \
       --dest-file $(MOZ_TEST_PACKAGES_FILE) \
       $(call PKG_ARG,common) \
-      $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
+      $(foreach pkg,$(ALL_TEST_PKGS),$(call PKG_ARG,$(pkg)))
 
 package-tests-prepare-dest:
 	@rm -f '$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)'
 	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
 
 package-tests-mozharness: package-tests-prepare-dest
-	$(call py_action,test_archive,mozharness $(abspath $(DIST))/$(PKG_PATH)/mozharness.zip)
+	$(call py_action,test_archive, \
+		mozharness \
+		$(abspath $(DIST))/$(PKG_PATH)/mozharness.zip)
 package-tests: package-tests-mozharness
 
 package-tests-common: stage-all package-tests-prepare-dest
 	cd $(abspath $(PKG_STAGE)) && \
 	  zip -rq9D '$(abspath $(DIST))/$(PKG_PATH)$(TEST_PACKAGE)' \
-	  * -x \*/.mkdir.done \*.pyc $(foreach name,$(TEST_PKGS),$(name)\*)
+	  * -x \*/.mkdir.done \*.pyc $(foreach name,$(ALL_TEST_PKGS),$(name)\*)
 package-tests: package-tests-common
 
 define package_archive
@@ -450,6 +457,16 @@ endef
 
 $(foreach name,$(TEST_PKGS),$(eval $(call package_archive,$(name))))
 
+define python_test_archive
+package-tests-$(1): stage-all package-tests-prepare-dest
+	$$(call py_action,test_archive, \
+		$(1) \
+		$$(abspath $$(DIST))/$$(PKG_PATH)/$$(PKG_BASENAME).$(1).tests.zip)
+package-tests: package-tests-$(1)
+endef
+
+$(foreach name,$(PYTHON_TEST_PKGS),$(eval $(call python_test_archive,$(name))))
+
 ifeq ($(MOZ_BUILD_APP),mobile/android)
 stage-all: stage-android
 stage-all: stage-instrumentation-tests

testing/xpcshell/Makefile.in

@@ -42,5 +42,4 @@ stage-package:
 	@(cd $(topsrcdir)/build && tar $(TAR_CREATE_FLAGS) - $(EXTRA_BUILD_FILES)) | (cd $(PKG_STAGE)/xpcshell && tar -xf -)
 	@cp $(DEPTH)/mozinfo.json $(PKG_STAGE)/xpcshell
 	@cp $(DEPTH)/build/automation.py $(PKG_STAGE)/xpcshell
-	(cd $(DEPTH)/_tests/xpcshell/ && tar $(TAR_CREATE_FLAGS) - *) | (cd $(PKG_STAGE)/xpcshell/tests && tar -xf -)
 	@(cd $(DIST)/bin/components && tar $(TAR_CREATE_FLAGS) - $(TEST_HARNESS_COMPONENTS)) | (cd $(PKG_STAGE)/bin/components && tar -xf -)