Bug 1230060 - Add a --track option to process_install_manifest. r=gps

There are currently two operating modes for process_install_manifest:
- default, which removes any file in the destination directory that is not
  in the install manifest
- --no-remove, which doesn't do the above.

While install manifests also have the ability to deal with files that may
be left in the destination directory some other way, that requires knowing the
list of those files in advance, which is not always possible.

For instance:
- with the FasterMake build backend, install manifests are split such that
  there is one manifest per application of addon directory (to allow more
  parallelism), which means there is one for dist/bin and one for several
  of its sub-directories.
- With --disable-compile-environment combined with artefacts, the backends
  are not aware of e.g. all the libraries and executables that end up in
  dist/bin.

If we want to properly remove files when they are removed from moz.build
or jar.mn, we can't use --no-remove, but the alternative would remove those
files

So add an option that keeps a list of all the files that were installed as
part of processing the given install manifest(s). That information is simply
a dump of the install manifest, which, while it contains more information
than currently required, will allow to do smarter things in the future.

python/mozbuild/mozbuild/action/process_install_manifest.py

@@ -5,8 +5,16 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
+import os
 import sys
-from mozpack.copier import FileCopier
+from mozpack.copier import (
+    FileCopier,
+    FileRegistry,
+)
+from mozpack.files import (
+    BaseFile,
+    FileFinder,
+)
 from mozpack.manifests import InstallManifest
 
 
@@ -14,22 +22,49 @@ COMPLETE = 'From {dest}: Kept {existing} existing; Added/updated {updated}; ' \
     'Removed {rm_files} files and {rm_dirs} directories.'
 
 
-def process_manifest(destdir, paths,
+def process_manifest(destdir, paths, track=None,
         remove_unaccounted=True,
         remove_all_directory_symlinks=True,
         remove_empty_directories=True,
         defines={}):
+
+    if track:
+        if os.path.exists(track):
+            # We use the same format as install manifests for the tracking
+            # data.
+            manifest = InstallManifest(path=track)
+            remove_unaccounted = FileRegistry()
+            dummy_file = BaseFile()
+
+            finder = FileFinder(destdir, find_executables=False,
+                                find_dotfiles=True)
+            for dest in manifest._dests:
+                if '*' in dest:
+                    for p, f in finder.find(dest):
+                        remove_unaccounted.add(p, dummy_file)
+                else:
+                    remove_unaccounted.add(dest, dummy_file)
+        else:
+            # If tracking is enabled and there is no file, we don't want to
+            # be removing anything.
+            remove_unaccounted=False
+
     manifest = InstallManifest()
     for path in paths:
         manifest |= InstallManifest(path=path)
 
     copier = FileCopier()
     manifest.populate_registry(copier, defines_override=defines)
-    return copier.copy(destdir,
+    result = copier.copy(destdir,
         remove_unaccounted=remove_unaccounted,
         remove_all_directory_symlinks=remove_all_directory_symlinks,
         remove_empty_directories=remove_empty_directories)
 
+    if track:
+        manifest.write(path=track)
+
+    return result
+
 
 class DefinesAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string):
@@ -59,6 +94,8 @@ def main(argv):
         help='Do not remove all directory symlinks from destination.')
     parser.add_argument('--no-remove-empty-directories', action='store_true',
         help='Do not remove empty directories from destination.')
+    parser.add_argument('--track', metavar="PATH",
+        help='Use installed files tracking information from the given path.')
     parser.add_argument('-D', action=DefinesAction,
         dest='defines', metavar="VAR[=VAL]",
         help='Define a variable to override what is specified in the manifest')
@@ -66,7 +103,7 @@ def main(argv):
     args = parser.parse_args(argv)
 
     result = process_manifest(args.destdir, args.manifests,
-        remove_unaccounted=not args.no_remove,
+        track=args.track, remove_unaccounted=not args.no_remove,
         remove_all_directory_symlinks=not args.no_remove_all_directory_symlinks,
         remove_empty_directories=not args.no_remove_empty_directories,
         defines=args.defines)