diff --git a/toolkit/crashreporter/tools/symbolstore.py b/toolkit/crashreporter/tools/symbolstore.py index 30d3fe144f7..14bb962f5d3 100755 --- a/toolkit/crashreporter/tools/symbolstore.py +++ b/toolkit/crashreporter/tools/symbolstore.py @@ -14,7 +14,8 @@ # # Parameters accepted: # -c : Copy debug info files to the same directory structure -# as sym files +# as sym files. On Windows, this will also copy +# binaries into the symbol store. # -a "" : Run dump_syms -a for each space separated # cpu architecture in (only on OS X) # -s : Use as the top source directory to @@ -535,7 +536,7 @@ class Dumper: return "" # subclasses override this if they want to support this - def CopyDebug(self, file, debug_file, guid): + def CopyDebug(self, file, debug_file, guid, code_file, code_id): pass def Finish(self, stop_pool=True): @@ -609,6 +610,7 @@ class Dumper: result = { 'status' : False, 'after' : after, 'after_arg' : after_arg, 'files' : files } sourceFileStream = '' + code_id, code_file = None, None for file in files: # files is a tuple of files, containing fallbacks in case the first file doesn't process successfully try: @@ -653,6 +655,14 @@ class Dumper: (ver, checkout, source_file, revision) = filename.split(":", 3) sourceFileStream += sourcepath + "*" + source_file + '*' + revision + "\r\n" f.write("FILE %s %s\n" % (index, filename)) + elif line.startswith("INFO CODE_ID "): + # INFO CODE_ID code_id code_file + # This gives some info we can use to + # store binaries in the symbol store. + bits = line.rstrip().split(None, 3) + if len(bits) == 4: + code_id, code_file = bits[2:] + f.write(line) else: # pass through all other lines unchanged f.write(line) @@ -668,7 +678,8 @@ class Dumper: self.SourceServerIndexing(file, guid, sourceFileStream, vcs_root) # only copy debug the first time if we have multiple architectures if self.copy_debug and arch_num == 0: - self.CopyDebug(file, debug_file, guid) + self.CopyDebug(file, debug_file, guid, + code_file, code_id) except StopIteration: pass except Exception as e: @@ -720,26 +731,53 @@ class Dumper_Win32(Dumper): self.fixedFilenameCaseCache[file] = result return result - def CopyDebug(self, file, debug_file, guid): + def CopyDebug(self, file, debug_file, guid, code_file, code_id): + def compress(path): + compressed_file = path[:-1] + '_' + # ignore makecab's output + success = subprocess.call(["makecab.exe", "/D", + "CompressionType=LZX", "/D", + "CompressionMemory=21", + path, compressed_file], + stdout=open("NUL:","w"), + stderr=subprocess.STDOUT) + if success == 0 and os.path.exists(compressed_file): + os.unlink(path) + return True + return False + rel_path = os.path.join(debug_file, guid, debug_file).replace("\\", "/") full_path = os.path.normpath(os.path.join(self.symbol_path, rel_path)) shutil.copyfile(file, full_path) - # try compressing it - compressed_file = os.path.splitext(full_path)[0] + ".pd_" - # ignore makecab's output - success = subprocess.call(["makecab.exe", "/D", "CompressionType=LZX", "/D", - "CompressionMemory=21", - full_path, compressed_file], - stdout=open("NUL:","w"), stderr=subprocess.STDOUT) - if success == 0 and os.path.exists(compressed_file): - os.unlink(full_path) - self.output(sys.stdout, os.path.splitext(rel_path)[0] + ".pd_") + if compress(full_path): + self.output(sys.stdout, rel_path[:-1] + '_') else: self.output(sys.stdout, rel_path) - + + # Copy the binary file as well + if code_file and code_id: + full_code_path = os.path.join(os.path.dirname(file), + code_file) + if os.path.exists(full_code_path): + rel_path = os.path.join(code_file, + code_id, + code_file).replace("\\", "/") + full_path = os.path.normpath(os.path.join(self.symbol_path, + rel_path)) + try: + os.makedirs(os.path.dirname(full_path)) + except OSError as e: + if e.errno != errno.EEXIST: + raise + shutil.copyfile(full_code_path, full_path) + if compress(full_path): + self.output(sys.stdout, rel_path[:-1] + '_') + else: + self.output(sys.stdout, rel_path) + def SourceServerIndexing(self, debug_file, guid, sourceFileStream, vcs_root): # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing debug_file = os.path.abspath(debug_file) @@ -770,7 +808,7 @@ class Dumper_Linux(Dumper): return self.RunFileCommand(file).startswith("ELF") return False - def CopyDebug(self, file, debug_file, guid): + def CopyDebug(self, file, debug_file, guid, code_file, code_id): # We want to strip out the debug info, and add a # .gnu_debuglink section to the object, so the debugger can # actually load our debug info later. @@ -881,7 +919,7 @@ class Dumper_Mac(Dumper): result['files'] = (dsymbundle, file) return result - def CopyDebug(self, file, debug_file, guid): + def CopyDebug(self, file, debug_file, guid, code_file, code_id): """ProcessFiles has already produced a dSYM bundle, so we should just copy that to the destination directory. However, we'll package it into a .tar.bz2 because the debug symbols are pretty huge, and diff --git a/toolkit/crashreporter/tools/unit-symbolstore.py b/toolkit/crashreporter/tools/unit-symbolstore.py index 1d9f276d92a..7d6c7d7634c 100644 --- a/toolkit/crashreporter/tools/unit-symbolstore.py +++ b/toolkit/crashreporter/tools/unit-symbolstore.py @@ -105,54 +105,33 @@ class TestExclude(HelperMixin, unittest.TestCase): expected.sort() self.assertEqual(processed, expected) -def popen_factory(stdouts): - """ - Generate a class that can mock subprocess.Popen. |stdouts| is an iterable that - should return an iterable for the stdout of each process in turn. - """ - class mock_popen(object): - def __init__(self, args, *args_rest, **kwargs): - self.stdout = stdouts.next() - - def wait(self): - return 0 - return mock_popen - -def mock_dump_syms(module_id, filename): - return ["MODULE os x86 %s %s" % (module_id, filename), +def mock_dump_syms(module_id, filename, extra=[]): + return ["MODULE os x86 %s %s" % (module_id, filename) + ] + extra + [ "FILE 0 foo.c", "PUBLIC xyz 123"] -class TestCopyDebugUniversal(HelperMixin, unittest.TestCase): - """ - Test that CopyDebug does the right thing when dumping multiple architectures. - """ + +class TestCopyDebug(HelperMixin, unittest.TestCase): def setUp(self): HelperMixin.setUp(self) self.symbol_dir = tempfile.mkdtemp() - self._subprocess_call = subprocess.call - subprocess.call = self.mock_call - self._subprocess_popen = subprocess.Popen - subprocess.Popen = popen_factory(self.next_mock_stdout()) + self.mock_call = patch("subprocess.call").start() self.stdouts = [] - self._shutil_rmtree = shutil.rmtree - shutil.rmtree = self.mock_rmtree + self.mock_popen = patch("subprocess.Popen").start() + stdout_iter = self.next_mock_stdout() + def next_popen(*args, **kwargs): + m = mock.MagicMock() + m.stdout = stdout_iter.next() + m.wait.return_value = 0 + return m + self.mock_popen.side_effect = next_popen + shutil.rmtree = patch("shutil.rmtree").start() def tearDown(self): HelperMixin.tearDown(self) - shutil.rmtree = self._shutil_rmtree + patch.stopall() shutil.rmtree(self.symbol_dir) - subprocess.call = self._subprocess_call - subprocess.Popen = self._subprocess_popen - - def mock_rmtree(self, path): - pass - - def mock_call(self, args, **kwargs): - if args[0].endswith("dsymutil"): - filename = args[-1] - os.makedirs(filename + ".dSYM") - return 0 def next_mock_stdout(self): if not self.stdouts: @@ -166,11 +145,16 @@ class TestCopyDebugUniversal(HelperMixin, unittest.TestCase): per file. """ copied = [] - def mock_copy_debug(filename, debug_file, guid): + def mock_copy_debug(filename, debug_file, guid, code_file, code_id): copied.append(filename[len(self.symbol_dir):] if filename.startswith(self.symbol_dir) else filename) self.add_test_files(add_extension(["foo"])) self.stdouts.append(mock_dump_syms("X" * 33, add_extension(["foo"])[0])) self.stdouts.append(mock_dump_syms("Y" * 33, add_extension(["foo"])[0])) + def mock_dsymutil(args, **kwargs): + filename = args[-1] + os.makedirs(filename + ".dSYM") + return 0 + self.mock_call.side_effect = mock_dsymutil d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms", symbol_path=self.symbol_dir, copy_debug=True, @@ -180,6 +164,29 @@ class TestCopyDebugUniversal(HelperMixin, unittest.TestCase): d.Finish(stop_pool=False) self.assertEqual(1, len(copied)) + def test_copy_debug_copies_binaries(self): + """ + Test that CopyDebug copies binaries as well on Windows. + """ + test_file = os.path.join(self.test_dir, 'foo.pdb') + write_pdb(test_file) + code_file = 'foo.dll' + code_id = 'abc123' + self.stdouts.append(mock_dump_syms('X' * 33, 'foo.pdb', + ['INFO CODE_ID %s %s' % (code_id, code_file)])) + def mock_compress(args, **kwargs): + filename = args[-1] + open(filename, 'w').write('stuff') + return 0 + self.mock_call.side_effect = mock_compress + d = symbolstore.Dumper_Win32(dump_syms='dump_syms', + symbol_path=self.symbol_dir, + copy_debug=True) + d.FixFilenameCase = lambda f: f + d.Process(self.test_dir) + d.Finish(stop_pool=False) + self.assertTrue(os.path.isfile(os.path.join(self.symbol_dir, code_file, code_id, code_file[:-1] + '_'))) + class TestGetVCSFilename(HelperMixin, unittest.TestCase): def setUp(self): HelperMixin.setUp(self)