diff --git a/python/mozbuild/mozpack/files.py b/python/mozbuild/mozpack/files.py index 5c7b21cc903..f7e578c5c82 100644 --- a/python/mozbuild/mozpack/files.py +++ b/python/mozbuild/mozpack/files.py @@ -2,9 +2,12 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import errno import os import re import shutil +import stat +import uuid from mozpack.executables import ( is_executable, may_strip, @@ -159,6 +162,100 @@ class ExecutableFile(File): return True +class AbsoluteSymlinkFile(File): + '''File class that is copied by symlinking (if available). + + This class only works if the target path is absolute. + ''' + + def __init__(self, path): + if not os.path.isabs(path): + raise ValueError('Symlink target not absolute: %s' % path) + + File.__init__(self, path) + + def copy(self, dest, skip_if_older=True): + assert isinstance(dest, basestring) + + # The logic in this function is complicated by the fact that symlinks + # aren't universally supported. So, where symlinks aren't supported, we + # fall back to file copying. Keep in mind that symlink support is + # per-filesystem, not per-OS. + + # Handle the simple case where symlinks are definitely not supported by + # falling back to file copy. + if not hasattr(os, 'symlink'): + return File.copy(self, dest, skip_if_older=skip_if_older) + + # Always verify the symlink target path exists. + if not os.path.exists(self.path): + raise ErrorMessage('Symlink target path does not exist: %s' % self.path) + + st = None + + try: + st = os.lstat(dest) + except OSError as ose: + if ose.errno != errno.ENOENT: + raise + + # If the dest is a symlink pointing to us, we have nothing to do. + # If it's the wrong symlink, the filesystem must support symlinks, + # so we replace with a proper symlink. + if st and stat.S_ISLNK(st.st_mode): + link = os.readlink(dest) + if link == self.path: + return False + + os.remove(dest) + os.symlink(self.path, dest) + return True + + # If the destination doesn't exist, we try to create a symlink. If that + # fails, we fall back to copy code. + if not st: + try: + os.symlink(self.path, dest) + return True + except OSError: + return File.copy(self, dest, skip_if_older=skip_if_older) + + # Now the complicated part. If the destination exists, we could be + # replacing a file with a symlink. Or, the filesystem may not support + # symlinks. We want to minimize I/O overhead for performance reasons, + # so we keep the existing destination file around as long as possible. + # A lot of the system calls would be eliminated if we cached whether + # symlinks are supported. However, even if we performed a single + # up-front test of whether the root of the destination directory + # supports symlinks, there's no guarantee that all operations for that + # dest (or source) would be on the same filesystem and would support + # symlinks. + # + # Our strategy is to attempt to create a new symlink with a random + # name. If that fails, we fall back to copy mode. If that works, we + # remove the old destination and move the newly-created symlink into + # its place. + + temp_dest = os.path.join(os.path.dirname(dest), str(uuid.uuid4())) + try: + os.symlink(self.path, temp_dest) + # TODO Figure out exactly how symlink creation fails and only trap + # that. + except EnvironmentError: + return File.copy(self, dest, skip_if_older=skip_if_older) + + # If removing the original file fails, don't forget to clean up the + # temporary symlink. + try: + os.remove(dest) + except EnvironmentError: + os.remove(temp_dest) + raise + + os.rename(temp_dest, dest) + return True + + class GeneratedFile(BaseFile): ''' File class for content with no previous existence on the filesystem. diff --git a/python/mozbuild/mozpack/test/test_files.py b/python/mozbuild/mozpack/test/test_files.py index ad6d9e8722b..9af482f63c0 100644 --- a/python/mozbuild/mozpack/test/test_files.py +++ b/python/mozbuild/mozpack/test/test_files.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from mozpack.files import ( + AbsoluteSymlinkFile, Dest, File, GeneratedFile, @@ -216,6 +217,112 @@ class TestFile(TestWithTmpDir): self.assertEqual('fooo', open(dest, 'rb').read()) +class TestAbsoluteSymlinkFile(TestWithTmpDir): + def setUp(self): + TestWithTmpDir.setUp(self) + + self.symlink_supported = False + + if not hasattr(os, 'symlink'): + return + + dummy_path = self.tmppath('dummy_file') + with open(dummy_path, 'a'): + pass + + try: + os.symlink(dummy_path, self.tmppath('dummy_symlink')) + os.remove(self.tmppath('dummy_symlink')) + except EnvironmentError: + pass + finally: + os.remove(dummy_path) + + def test_absolute_relative(self): + AbsoluteSymlinkFile('/foo') + + with self.assertRaisesRegexp(ValueError, 'Symlink target not absolute'): + AbsoluteSymlinkFile('./foo') + + def test_symlink_file(self): + source = self.tmppath('test_path') + with open(source, 'wt') as fh: + fh.write('Hello world') + + s = AbsoluteSymlinkFile(source) + dest = self.tmppath('symlink') + self.assertTrue(s.copy(dest)) + + if self.symlink_supported: + self.assertTrue(os.path.islink(dest)) + link = os.readlink(dest) + self.assertEqual(link, source) + else: + self.assertTrue(os.path.isfile(dest)) + content = open(dest).read() + self.assertEqual(content, 'Hello world') + + def test_replace_file_with_symlink(self): + # If symlinks are supported, an existing file should be replaced by a + # symlink. + source = self.tmppath('test_path') + with open(source, 'wt') as fh: + fh.write('source') + + dest = self.tmppath('dest') + with open(dest, 'a'): + pass + + s = AbsoluteSymlinkFile(source) + s.copy(dest, skip_if_older=False) + + if self.symlink_supported: + self.assertTrue(os.path.islink(dest)) + link = os.readlink(dest) + self.assertEqual(link, source) + else: + self.assertTrue(os.path.isfile(dest)) + content = open(dest).read() + self.assertEqual(content, 'source') + + def test_replace_symlink(self): + if not self.symlink_supported: + return + + source= self.tmppath('source') + dest = self.tmppath('dest') + + os.symlink(self.tmppath('bad'), dest) + self.assertTrue(os.path.islink(dest)) + + s = AbsoluteSymlinkFile(source) + self.assertTrue(s.copy(dest)) + + self.assertTrue(os.path.islink(dest)) + link = os.readlink(dest) + self.assertEqual(link, source) + + def test_noop(self): + if not hasattr(os, 'symlink'): + return + + source = self.tmppath('source') + dest = self.tmppath('dest') + + with open(source, 'a'): + pass + + os.symlink(source, dest) + link = os.readlink(dest) + self.assertEqual(link, source) + + s = AbsoluteSymlinkFile(source) + self.assertFalse(s.copy(dest)) + + link = os.readlink(dest) + self.assertEqual(link, source) + + class TestGeneratedFile(TestWithTmpDir): def test_generated_file(self): '''