import os import math import mozinfo class Bisect(object): "Class for creating, bisecting and summarizing for --bisect-chunk option." def __init__(self, harness): super(Bisect, self).__init__() self.summary = [] self.contents = {} self.testRoot = harness.testRoot self.testRootAbs = harness.testRootAbs def setup(self, tests): "This method is used to initialize various variables that are required for test bisection" status = 0 self.contents.clear() # We need totalTests key in contents for sanity check self.contents['totalTests'] = tests self.contents['tests'] = tests self.contents['loop'] = 0 return status def reset(self, expectedError, result): "This method is used to initialize self.expectedError and self.result for each loop in runtests." self.expectedError = expectedError self.result = result def get_test_chunk(self, options, tests): "This method is used to return the chunk of test that is to be run" if not options.totalChunks or not options.thisChunk: return tests # The logic here is same as chunkifyTests.js, we need this for bisecting tests. if options.chunkByDir: tests_by_dir = {} test_dirs = [] for test in tests: directory = test.split("/") directory = directory[0:min(options.chunkByDir, len(directory)-1)] directory = "/".join(directory) if not directory in tests_by_dir: tests_by_dir[directory] = [test] test_dirs.append(directory) else: tests_by_dir[directory].append(test) tests_per_chunk = float(len(test_dirs)) / options.totalChunks start = int(round((options.thisChunk-1) * tests_per_chunk)) end = int(round((options.thisChunk) * tests_per_chunk)) test_dirs = test_dirs[start:end] return_tests = [] for directory in test_dirs: return_tests += tests_by_dir[directory] else: tests_per_chunk = float(len(tests)) / options.totalChunks start = int(round((options.thisChunk-1) * tests_per_chunk)) end = int(round(options.thisChunk * tests_per_chunk)) return_tests = tests[start:end] options.totalChunks = None options.thisChunk = None options.chunkByDir = None return return_tests def get_tests_for_bisection(self, options, tests): "Make a list of tests for bisection from a given list of tests" tests = self.get_test_chunk(options, tests) bisectlist = [] for test in tests: bisectlist.append(test) if test.endswith(options.bisectChunk): break return bisectlist def pre_test(self, options, tests, status): "This method is used to call other methods for setting up variables and getting the list of tests for bisection." if options.bisectChunk == "default": return tests # The second condition in 'if' is required to verify that the failing test is the last one. elif 'loop' not in self.contents or not self.contents['tests'][-1].endswith(options.bisectChunk): tests = self.get_tests_for_bisection(options, tests) status = self.setup(tests) return self.next_chunk_binary(options, status) def post_test(self, options, expectedError, result): "This method is used to call other methods to summarize results and check whether a sanity check is done or not." self.reset(expectedError, result) status = self.summarize_chunk(options) # Check whether sanity check has to be done. Also it is necessary to check whether options.bisectChunk is present # in self.expectedError as we do not want to run if it is "default". if status == -1 and options.bisectChunk in self.expectedError: # In case we have a debug build, we don't want to run a sanity check, will take too much time. if mozinfo.info['debug']: return status testBleedThrough = self.contents['testsToRun'][0] tests = self.contents['totalTests'] tests.remove(testBleedThrough) # To make sure that the failing test is dependent on some other test. if options.bisectChunk in testBleedThrough: return status status = self.setup(tests) self.summary.append("Sanity Check:") return status def next_chunk_reverse(self, options, status): "This method is used to bisect the tests in a reverse search fashion." # Base Cases. if self.contents['loop'] <= 1: self.contents['testsToRun'] = self.contents['tests'] if self.contents['loop'] == 1: self.contents['testsToRun'] = [self.contents['tests'][-1]] self.contents['loop'] += 1 return self.contents['testsToRun'] if 'result' in self.contents: if self.contents['result'] == "PASS": chunkSize = self.contents['end'] - self.contents['start'] self.contents['end'] = self.contents['start'] - 1 self.contents['start'] = self.contents['end'] - chunkSize # self.contents['result'] will be expected error only if it fails. elif self.contents['result'] == "FAIL": self.contents['tests'] = self.contents['testsToRun'] status = 1 # for initializing # initialize if status: totalTests = len(self.contents['tests']) chunkSize = int(math.ceil(totalTests / 10.0)) self.contents['start'] = totalTests - chunkSize - 1 self.contents['end'] = totalTests - 2 start = self.contents['start'] end = self.contents['end'] + 1 self.contents['testsToRun'] = self.contents['tests'][start:end] self.contents['testsToRun'].append(self.contents['tests'][-1]) self.contents['loop'] += 1 return self.contents['testsToRun'] def next_chunk_binary(self, options, status): "This method is used to bisect the tests in a binary search fashion." # Base cases. if self.contents['loop'] <= 1: self.contents['testsToRun'] = self.contents['tests'] if self.contents['loop'] == 1: self.contents['testsToRun'] = [self.contents['tests'][-1]] self.contents['loop'] += 1 return self.contents['testsToRun'] # Initialize the contents dict. if status: totalTests = len(self.contents['tests']) self.contents['start'] = 0 self.contents['end'] = totalTests - 2 mid = (self.contents['start'] + self.contents['end']) / 2 if 'result' in self.contents: if self.contents['result'] == "PASS": self.contents['end'] = mid elif self.contents['result'] == "FAIL": self.contents['start'] = mid + 1 mid = (self.contents['start'] + self.contents['end']) / 2 start = mid + 1 end = self.contents['end'] + 1 self.contents['testsToRun'] = self.contents['tests'][start:end] if not self.contents['testsToRun']: self.contents['testsToRun'].append(self.contents['tests'][mid]) self.contents['testsToRun'].append(self.contents['tests'][-1]) self.contents['loop'] += 1 return self.contents['testsToRun'] def summarize_chunk(self, options): "This method is used summarize the results after the list of tests is run." if options.bisectChunk == "default": # if no expectedError that means all the tests have successfully passed. if len(self.expectedError) == 0: return -1 options.bisectChunk = self.expectedError.keys()[0] self.summary.append("\tFound Error in test: %s" % options.bisectChunk) return 0 # If options.bisectChunk is not in self.result then we need to move to the next run. if options.bisectChunk not in self.result: return -1 self.summary.append("\tPass %d:" % self.contents['loop']) if len(self.contents['testsToRun']) > 1: self.summary.append("\t\t%d test files(start,end,failing). [%s, %s, %s]" % (len(self.contents['testsToRun']), self.contents['testsToRun'][0], self.contents['testsToRun'][-2], self.contents['testsToRun'][-1])) else: self.summary.append("\t\t1 test file [%s]" % self.contents['testsToRun'][0]) if self.result[options.bisectChunk] == "PASS": self.summary.append("\t\tno failures found.") if self.contents['loop'] == 1: status = -1 elif self.contents['loop'] == 2: status = 1 else: self.contents['result'] = "PASS" status = 0 elif self.result[options.bisectChunk] == "FAIL": if 'expectedError' not in self.contents: self.summary.append("\t\t%s failed." % self.contents['testsToRun'][-1]) self.contents['expectedError'] = self.expectedError[options.bisectChunk] status = 0 elif self.expectedError[options.bisectChunk] == self.contents['expectedError']: self.summary.append("\t\t%s failed with expected error." % self.contents['testsToRun'][-1]) self.contents['result'] = "FAIL" status = 0 # This code checks for test-bleedthrough. Should work for any algorithm. numberOfTests = len(self.contents['testsToRun']) if numberOfTests < 3: # This means that only 2 tests are run. Since the last test is the failing test itself therefore the bleedthrough test is the first test self.summary.append("TEST-BLEEDTHROUGH - found failure, %s" % self.contents['testsToRun'][0]) status = -1 else: self.summary.append("\t\t%s failed with different error." % self.contents['testsToRun'][-1]) status = -1 return status def print_summary(self): "This method is used to print the recorded summary." print "Bisection summary:" for line in self.summary: print line