Bug 1185971 - mozfile: add a mozfile.move function, and bump release 1.2. r=ahal

testing/mozbase/docs/mozfile.rst

@@ -6,5 +6,5 @@ tasks in automated testing, such as extracting files or recursively removing
 directories.
 
 .. automodule:: mozfile
-   :members: extract, extract_tarball, extract_zip, remove
+   :members: extract, extract_tarball, extract_zip, move, remove
 

testing/mozbase/mozfile/mozfile/mozfile.py

@@ -20,6 +20,7 @@ __all__ = ['extract_tarball',
            'extract',
            'is_url',
            'load',
+           'move',
            'remove',
            'rmtree',
            'tree',
@@ -134,6 +135,35 @@ def rmtree(dir):
     return remove(dir)
 
 
+def _call_windows_retry(func, args=(), retry_max=5, retry_delay=0.5):
+    """
+    It's possible to see spurious errors on Windows due to various things
+    keeping a handle to the directory open (explorer, virus scanners, etc)
+    So we try a few times if it fails with a known error.
+    """
+    retry_count = 0
+    while True:
+        try:
+            func(*args)
+        except OSError, e:
+            # Error codes are defined in:
+            # http://docs.python.org/2/library/errno.html#module-errno
+            if e.errno not in (errno.EACCES, errno.ENOTEMPTY):
+                raise
+
+            if retry_count == retry_max:
+                raise
+
+            retry_count += 1
+
+            print '%s() failed for "%s". Reason: %s (%s). Retrying...' % \
+                    (func.__name__, args, e.strerror, e.errno)
+            time.sleep(retry_delay)
+        else:
+            # If no exception has been thrown it should be done
+            break
+
+
 def remove(path):
     """Removes the specified file, link, or directory tree.
 
@@ -154,37 +184,13 @@ def remove(path):
 
     import shutil
 
-    def _call_with_windows_retry(func, args=(), retry_max=5, retry_delay=0.5):
-        """
-        It's possible to see spurious errors on Windows due to various things
-        keeping a handle to the directory open (explorer, virus scanners, etc)
-        So we try a few times if it fails with a known error.
-        """
-        retry_count = 0
-        while True:
-            try:
-                func(*args)
-            except OSError, e:
-                # The file or directory to be removed doesn't exist anymore
-                if e.errno == errno.ENOENT:
-                    break
-
-                # Error codes are defined in:
-                # http://docs.python.org/2/library/errno.html#module-errno
-                if e.errno not in [errno.EACCES, errno.ENOTEMPTY]:
-                    raise
-
-                if retry_count == retry_max:
-                    raise
-
-                retry_count += 1
-
-                print '%s() failed for "%s". Reason: %s (%s). Retrying...' % \
-                        (func.__name__, args, e.strerror, e.errno)
-                time.sleep(retry_delay)
-            else:
-                # If no exception has been thrown it should be done
-                break
+    def _call_with_windows_retry(*args, **kwargs):
+        try:
+            _call_windows_retry(*args, **kwargs)
+        except OSError, e:
+            # The file or directory to be removed doesn't exist anymore
+            if e.errno != errno.ENOENT:
+                raise
 
     def _update_permissions(path):
         """Sets specified pemissions depending on filetype"""
@@ -224,6 +230,18 @@ def remove(path):
         _call_with_windows_retry(shutil.rmtree, (path,))
 
 
+def move(src, dst):
+    """
+    Move a file or directory path.
+
+    This is a replacement for shutil.move that works better under windows,
+    retrying operations on some known errors due to various things keeping
+    a handle on file paths.
+    """
+    import shutil
+    _call_windows_retry(shutil.move, (src, dst))
+
+
 def depth(directory):
     """returns the integer depth of a directory or path relative to '/' """
 

testing/mozbase/mozfile/setup.py

@@ -5,7 +5,7 @@
 from setuptools import setup
 
 PACKAGE_NAME = 'mozfile'
-PACKAGE_VERSION = '1.1'
+PACKAGE_VERSION = '1.2'
 
 setup(name=PACKAGE_NAME,
       version=PACKAGE_VERSION,

testing/mozbase/mozfile/tests/manifest.ini

@@ -1,6 +1,6 @@
 [test_extract.py]
 [test_load.py]
-[test_remove.py]
+[test_move_remove.py]
 [test_tempdir.py]
 [test_tempfile.py]
 [test_url.py]

testing/mozbase/mozfile/tests/test_move_remove.py

@@ -7,6 +7,7 @@ import threading
 import time
 import unittest
 import errno
+from contextlib import contextmanager
 
 import mozfile
 import mozinfo
@@ -27,12 +28,14 @@ class FileOpenCloseThread(threading.Thread):
     """Helper thread for asynchronous file handling"""
     def __init__(self, path, delay, delete=False):
         threading.Thread.__init__(self)
+        self.file_opened = threading.Event()
         self.delay = delay
         self.path = path
         self.delete = delete
 
     def run(self):
         with open(self.path):
+            self.file_opened.set()
             time.sleep(self.delay)
         if self.delete:
             try:
@@ -41,6 +44,17 @@ class FileOpenCloseThread(threading.Thread):
                 pass
 
 
+@contextmanager
+def wait_file_opened_in_thread(*args, **kwargs):
+    thread = FileOpenCloseThread(*args, **kwargs)
+    thread.start()
+    thread.file_opened.wait()
+    try:
+        yield thread
+    finally:
+        thread.join()
+
+
 class MozfileRemoveTestCase(unittest.TestCase):
     """Test our ability to remove directories and files"""
 
@@ -90,13 +104,10 @@ class MozfileRemoveTestCase(unittest.TestCase):
         """Test removing a file in use with retry"""
         filepath = os.path.join(self.tempdir, *stubs.files[1])
 
-        thread = FileOpenCloseThread(filepath, 1)
-        thread.start()
-
-        # Wait a bit so we can be sure the file has been opened
-        time.sleep(.5)
-        mozfile.remove(filepath)
-        thread.join()
+        with wait_file_opened_in_thread(filepath, 0.2):
+            # on windows first attempt will fail,
+            # and it will be retried until the thread leave the handle
+            mozfile.remove(filepath)
 
         # Check deletion was successful
         self.assertFalse(os.path.exists(filepath))
@@ -105,14 +116,10 @@ class MozfileRemoveTestCase(unittest.TestCase):
         """Test removing a meanwhile removed file with retry"""
         filepath = os.path.join(self.tempdir, *stubs.files[1])
 
-        thread = FileOpenCloseThread(filepath, .8, True)
-        thread.start()
-
-        # Wait a bit so we can be sure the file has been opened and gets deleted
-        # while remove() waits for the next retry
-        time.sleep(.5)
-        mozfile.remove(filepath)
-        thread.join()
+        with wait_file_opened_in_thread(filepath, 0.2, True):
+            # on windows first attempt will fail, and before
+            # the retry the opened file will be deleted in the thread
+            mozfile.remove(filepath)
 
         # Check deletion was successful
         self.assertFalse(os.path.exists(filepath))
@@ -192,5 +199,32 @@ class MozfileRemoveTestCase(unittest.TestCase):
                 self.fail("removing non existing path must not raise error")
             raise
 
+
+class MozFileMoveTestCase(unittest.TestCase):
+    def setUp(self):
+        # Generate a stub
+        self.tempdir = stubs.create_stub()
+        self.addCleanup(mozfile.rmtree, self.tempdir)
+
+    def test_move_file(self):
+        file_path = os.path.join(self.tempdir, *stubs.files[1])
+        moved_path = file_path + '.moved'
+        self.assertTrue(os.path.isfile(file_path))
+        self.assertFalse(os.path.exists(moved_path))
+        mozfile.move(file_path, moved_path)
+        self.assertFalse(os.path.exists(file_path))
+        self.assertTrue(os.path.isfile(moved_path))
+
+    def test_move_file_with_retry(self):
+        file_path = os.path.join(self.tempdir, *stubs.files[1])
+        moved_path = file_path + '.moved'
+
+        with wait_file_opened_in_thread(file_path, 0.2):
+            # first move attempt should fail on windows and be retried
+            mozfile.move(file_path, moved_path)
+            self.assertFalse(os.path.exists(file_path))
+            self.assertTrue(os.path.isfile(moved_path))
+
+
 if __name__ == '__main__':
     unittest.main()