diff --git a/Makefile.in b/Makefile.in index b3a4c6312bc..f00b97fded0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -38,29 +38,15 @@ DIST_GARBAGE = config.cache config.log config.status* config-defs.h \ $(topsrcdir)/.mozconfig.mk $(topsrcdir)/.mozconfig.out ifndef MOZ_PROFILE_USE -# One of the first things we do in the build is purge "unknown" files -# from the object directory. This serves two purposes: -# -# 1) Remove files from a previous build no longer accounted for in -# this build configuration. -# -# 2) Work around poor build system dependencies by forcing some -# rebuilds. -# -# Ideally #2 does not exist. Our reliance on this aspect should diminish -# over time. -# -# moz.build backend generation simply installs a set of "manifests" into -# a common directory. Each manifest is responsible for defining files in -# a specific subdirectory of the object directory. The invoked Python -# script simply iterates over all the manifests, purging files as -# necessary. To manage new directories or add files to the manifests, -# modify the backend generator. -# # We need to explicitly put backend.RecursiveMakeBackend.built here # otherwise the rule in rules.mk doesn't run early enough. default alldep all:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built - $(PYTHON) $(topsrcdir)/config/purge_directories.py -d _build_manifests/purge . + $(RM) -r $(DIST)/sdk + $(RM) -r $(DIST)/include + $(RM) -r $(DIST)/private + $(RM) -r $(DIST)/public + $(RM) -r $(DIST)/bin + $(RM) -r _tests endif CLOBBER: $(topsrcdir)/CLOBBER diff --git a/config/purge_directories.py b/config/purge_directories.py deleted file mode 100644 index 899824b22d4..00000000000 --- a/config/purge_directories.py +++ /dev/null @@ -1,77 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# This script is used to purge a directory of unwanted files as defined by -# a manifest file. - -from __future__ import print_function, unicode_literals - -import argparse -import os -import sys -import threading - -from mozpack.manifests import PurgeManifest - -def do_purge(purger, dest, state): - state['result'] = purger.purge(dest) - -def process_manifest(topdir, manifest_path): - manifest = PurgeManifest.from_path(manifest_path) - purger = manifest.get_purger() - full = os.path.join(topdir, manifest.relpath) - - state = dict( - relpath=manifest.relpath, - result=None, - ) - - t = threading.Thread(target=do_purge, args=(purger, full, state)) - state['thread'] = t - t.start() - - return state - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Purge a directory of untracked files.') - - parser.add_argument('--directory', '-d', - help='Directory containing manifest files. Will process every file ' - 'in directory.') - parser.add_argument('topdir', - help='Top directory all paths are evaluated from.') - parser.add_argument('manifests', nargs='*', - help='List of manifest files defining purge operations to perform.') - - args = parser.parse_args() - - states = [] - - print('Purging unaccounted files from object directory...') - - # We perform purging using threads for performance reasons. Hopefully - # multiple I/O operations will be faster than just 1. - paths = [] - if args.directory: - for path in sorted(os.listdir(args.directory)): - paths.append(os.path.join(args.directory, path)) - - paths.extend(args.manifests) - - for path in paths: - states.append(process_manifest(args.topdir, path)) - - for state in states: - state['thread'].join() - print('Deleted %d files and %d directories from %s.' % ( - state['result'].removed_files_count, - state['result'].removed_directories_count, - state['relpath'] - )) - - print('Finished purging.') - - sys.exit(0) diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index 42cdd7551ef..839b5f22f03 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -9,9 +9,6 @@ import logging import os import types -from mozpack.copier import FilePurger -from mozpack.manifests import PurgeManifest - from .base import BuildBackend from ..frontend.data import ( ConfigFileSubstitution, @@ -130,15 +127,6 @@ class RecursiveMakeBackend(BuildBackend): self.backend_input_files.add(os.path.join(self.environment.topobjdir, 'config', 'autoconf.mk')) - self._purge_manifests = dict( - dist_bin=PurgeManifest(relpath='dist/bin'), - dist_include=PurgeManifest(relpath='dist/include'), - dist_private=PurgeManifest(relpath='dist/private'), - dist_public=PurgeManifest(relpath='dist/public'), - dist_sdk=PurgeManifest(relpath='dist/sdk'), - tests=PurgeManifest(relpath='_tests'), - ) - def _update_from_avoid_write(self, result): existed, updated = result @@ -264,8 +252,6 @@ class RecursiveMakeBackend(BuildBackend): self._update_from_avoid_write(mastermanifest.close()) self.summary.managed_count += 1 - self._write_purge_manifests() - def _process_directory_traversal(self, obj, backend_file): """Process a data.DirectoryTraversal instance.""" fh = backend_file.fh @@ -323,10 +309,6 @@ class RecursiveMakeBackend(BuildBackend): if strings: backend_file.write('%s += %s\n' % (export_name, ' '.join(strings))) - for s in strings: - p = '%s%s' % (namespace, s) - self._purge_manifests['dist_include'].add(p) - children = exports.get_children() for subdir in sorted(children): self._process_exports(children[subdir], backend_file, @@ -341,27 +323,3 @@ class RecursiveMakeBackend(BuildBackend): if obj.relativedir != '': manifest = '%s/%s' % (obj.relativedir, manifest) self.xpcshell_manifests.append(manifest) - - def _write_purge_manifests(self): - # We write out a "manifest" file for each directory that is to be - # purged. - # - # Ideally we have as few manifests as possible - ideally only 1. This - # will likely require all build metadata to be in emitted objects. - # We're not quite there yet, so we maintain multiple manifests. - man_dir = os.path.join(self.environment.topobjdir, '_build_manifests', - 'purge') - - # We have a purger for the manifests themselves to ensure we don't over - # purge if we delete a purge manifest. - purger = FilePurger() - - for k, manifest in self._purge_manifests.items(): - purger.add(k) - full = os.path.join(man_dir, k) - - fh = FileAvoidWrite(os.path.join(man_dir, k)) - manifest.write_fileobj(fh) - self._update_from_avoid_write(fh.close()) - - purger.purge(man_dir) diff --git a/python/mozbuild/mozbuild/test/backend/common.py b/python/mozbuild/mozbuild/test/backend/common.py index b9169c2d480..ec46d8661ba 100644 --- a/python/mozbuild/mozbuild/test/backend/common.py +++ b/python/mozbuild/mozbuild/test/backend/common.py @@ -89,17 +89,15 @@ class BackendTester(unittest.TestCase): config['substs'].append(('top_srcdir', srcdir)) return ConfigEnvironment(srcdir, objdir, **config) - def _emit(self, name, env=None): - if not env: - env = self._get_environment(name) - + def _emit(self, name): + env = self._get_environment(name) reader = BuildReader(env) emitter = TreeMetadataEmitter(env) return env, emitter.emit(reader.read_topsrcdir()) - def _consume(self, name, cls, env=None): - env, objs = self._emit(name, env=env) + def _consume(self, name, cls): + env, objs = self._emit(name) backend = cls(env) backend.consume(objs) diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index 99b6c94ef9f..48ebabbbe94 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -7,7 +7,6 @@ from __future__ import unicode_literals import os import time -from mozpack.manifests import PurgeManifest from mozunit import main from mozbuild.backend.configenvironment import ConfigEnvironment @@ -231,13 +230,6 @@ class TestRecursiveMakeBackend(BackendTester): 'EXPORTS_nspr/private += pprio.h', ]) - # EXPORTS files should appear in the dist_include purge manifest. - m = PurgeManifest.from_path(os.path.join(env.topobjdir, - '_build_manifests', 'purge', 'dist_include')) - self.assertIn('foo.h', m.entries) - self.assertIn('mozilla/mozilla1.h', m.entries) - self.assertIn('mozilla/dom/dom2.h', m.entries) - def test_xpcshell_manifests(self): """Ensure XPCSHELL_TESTS_MANIFESTS is written out correctly.""" env = self._consume('xpcshell_manifests', RecursiveMakeBackend) @@ -265,42 +257,5 @@ class TestRecursiveMakeBackend(BackendTester): '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.', ''] + ['[include:%s/xpcshell.ini]' % x for x in expected]) - def test_purge_manifests_written(self): - env = self._consume('stub0', RecursiveMakeBackend) - - purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge') - self.assertTrue(os.path.exists(purge_dir)) - - expected = [ - 'dist_bin', - 'dist_include', - 'dist_private', - 'dist_public', - 'dist_sdk', - 'tests', - ] - - for e in expected: - full = os.path.join(purge_dir, e) - self.assertTrue(os.path.exists(full)) - - m = PurgeManifest.from_path(os.path.join(purge_dir, 'dist_bin')) - self.assertEqual(m.relpath, 'dist/bin') - - def test_old_purge_manifest_deleted(self): - # Simulate a purge manifest from a previous backend version. Ensure it - # is deleted. - env = self._get_environment('stub0') - purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge') - manifest_path = os.path.join(purge_dir, 'old_manifest') - os.makedirs(purge_dir) - m = PurgeManifest() - m.write_file(manifest_path) - - self.assertTrue(os.path.exists(manifest_path)) - self._consume('stub0', RecursiveMakeBackend, env) - self.assertFalse(os.path.exists(manifest_path)) - - if __name__ == '__main__': main() diff --git a/python/mozbuild/mozpack/manifests.py b/python/mozbuild/mozpack/manifests.py deleted file mode 100644 index 352ea7a3bc2..00000000000 --- a/python/mozbuild/mozpack/manifests.py +++ /dev/null @@ -1,89 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import unicode_literals - -from .copier import FilePurger -import mozpack.path as mozpath - - -class UnreadablePurgeManifest(Exception): - """Error for failure when reading content of a serialized PurgeManifest.""" - - -class PurgeManifest(object): - """Describes actions to be used with a copier.FilePurger instance. - - This class facilitates serialization and deserialization of data used - to construct a copier.FilePurger and to perform a purge operation. - - The manifest contains a set of entries (paths that are accounted for and - shouldn't be purged) and a relative path. The relative path is optional and - can be used to e.g. have several manifest files in a directory be - dynamically applied to subdirectories under a common base directory. - - Don't be confused by the name of this class: entries are files that are - *not* purged. - """ - def __init__(self, relpath=''): - self.relpath = relpath - self.entries = set() - - def __eq__(self, other): - if not isinstance(other, PurgeManifest): - return False - - return other.relpath == self.relpath and other.entries == self.entries - - @staticmethod - def from_path(path): - with open(path, 'rt') as fh: - return PurgeManifest.from_fileobj(fh) - - @staticmethod - def from_fileobj(fh): - m = PurgeManifest() - - version = fh.readline().rstrip() - if version != '1': - raise UnreadablePurgeManifest('Unknown manifest version: ' % - version) - - m.relpath = fh.readline().rstrip() - - for entry in fh: - m.entries.add(entry.rstrip()) - - return m - - def add(self, path): - return self.entries.add(path) - - def write_file(self, path): - with open(path, 'wt') as fh: - return self.write_fileobj(fh) - - def write_fileobj(self, fh): - fh.write('1\n') - fh.write('%s\n' % self.relpath) - - # We write sorted so written output is consistent. - for entry in sorted(self.entries): - fh.write('%s\n' % entry) - - def get_purger(self, prepend_relpath=False): - """Obtain a FilePurger instance from this manifest. - - If :prepend_relpath is truish, the relative path in the manifest will - be prepended to paths added to the FilePurger. Otherwise, the raw paths - will be used. - """ - p = FilePurger() - for entry in self.entries: - if prepend_relpath: - entry = mozpath.join(self.relpath, entry) - - p.add(entry) - - return p diff --git a/python/mozbuild/mozpack/test/test_manifests.py b/python/mozbuild/mozpack/test/test_manifests.py deleted file mode 100644 index 9d956c82ed8..00000000000 --- a/python/mozbuild/mozpack/test/test_manifests.py +++ /dev/null @@ -1,48 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import unicode_literals - -import os -import unittest - -import mozunit - -from mozpack.manifests import ( - PurgeManifest, - UnreadablePurgeManifest, -) -from mozpack.test.test_files import TestWithTmpDir - - -class TestPurgeManifest(TestWithTmpDir): - def test_construct(self): - m = PurgeManifest() - self.assertEqual(m.relpath, '') - self.assertEqual(len(m.entries), 0) - - def test_serialization(self): - m = PurgeManifest(relpath='rel') - m.add('foo') - m.add('bar') - p = self.tmppath('m') - m.write_file(p) - - self.assertTrue(os.path.exists(p)) - - m2 = PurgeManifest.from_path(p) - self.assertEqual(m.relpath, m2.relpath) - self.assertEqual(m.entries, m2.entries) - self.assertEqual(m, m2) - - def test_unknown_version(self): - p = self.tmppath('bad') - - with open(p, 'wt') as fh: - fh.write('2\n') - fh.write('not relevant') - - with self.assertRaises(UnreadablePurgeManifest): - PurgeManifest.from_path(p) -