gecko/testing/mochitest/bisection.py

271 lines
11 KiB
Python

import math
import mozinfo
class Bisect(object):
"Class for creating, bisecting and summarizing for --bisect-chunk option."
def __init__(self, harness):
super(Bisect, self).__init__()
self.summary = []
self.contents = {}
self.repeat = 10
self.failcount = 0
self.max_failures = 3
def setup(self, tests):
"This method is used to initialize various variables that are required for test bisection"
status = 0
self.contents.clear()
# We need totalTests key in contents for sanity check
self.contents['totalTests'] = tests
self.contents['tests'] = tests
self.contents['loop'] = 0
return status
def reset(self, expectedError, result):
"This method is used to initialize self.expectedError and self.result for each loop in runtests."
self.expectedError = expectedError
self.result = result
def get_tests_for_bisection(self, options, tests):
"Make a list of tests for bisection from a given list of tests"
bisectlist = []
for test in tests:
bisectlist.append(test)
if test.endswith(options.bisectChunk):
break
return bisectlist
def pre_test(self, options, tests, status):
"This method is used to call other methods for setting up variables and getting the list of tests for bisection."
if options.bisectChunk == "default":
return tests
# The second condition in 'if' is required to verify that the failing
# test is the last one.
elif 'loop' not in self.contents or not self.contents['tests'][-1].endswith(options.bisectChunk):
tests = self.get_tests_for_bisection(options, tests)
status = self.setup(tests)
return self.next_chunk_binary(options, status)
def post_test(self, options, expectedError, result):
"This method is used to call other methods to summarize results and check whether a sanity check is done or not."
self.reset(expectedError, result)
status = self.summarize_chunk(options)
# Check whether sanity check has to be done. Also it is necessary to check whether options.bisectChunk is present
# in self.expectedError as we do not want to run if it is "default".
if status == -1 and options.bisectChunk in self.expectedError:
# In case we have a debug build, we don't want to run a sanity
# check, will take too much time.
if mozinfo.info['debug']:
return status
testBleedThrough = self.contents['testsToRun'][0]
tests = self.contents['totalTests']
tests.remove(testBleedThrough)
# To make sure that the failing test is dependent on some other
# test.
if options.bisectChunk in testBleedThrough:
return status
status = self.setup(tests)
self.summary.append("Sanity Check:")
return status
def next_chunk_reverse(self, options, status):
"This method is used to bisect the tests in a reverse search fashion."
# Base Cases.
if self.contents['loop'] <= 1:
self.contents['testsToRun'] = self.contents['tests']
if self.contents['loop'] == 1:
self.contents['testsToRun'] = [self.contents['tests'][-1]]
self.contents['loop'] += 1
return self.contents['testsToRun']
if 'result' in self.contents:
if self.contents['result'] == "PASS":
chunkSize = self.contents['end'] - self.contents['start']
self.contents['end'] = self.contents['start'] - 1
self.contents['start'] = self.contents['end'] - chunkSize
# self.contents['result'] will be expected error only if it fails.
elif self.contents['result'] == "FAIL":
self.contents['tests'] = self.contents['testsToRun']
status = 1 # for initializing
# initialize
if status:
totalTests = len(self.contents['tests'])
chunkSize = int(math.ceil(totalTests / 10.0))
self.contents['start'] = totalTests - chunkSize - 1
self.contents['end'] = totalTests - 2
start = self.contents['start']
end = self.contents['end'] + 1
self.contents['testsToRun'] = self.contents['tests'][start:end]
self.contents['testsToRun'].append(self.contents['tests'][-1])
self.contents['loop'] += 1
return self.contents['testsToRun']
def next_chunk_binary(self, options, status):
"This method is used to bisect the tests in a binary search fashion."
# Base cases.
if self.contents['loop'] <= 1:
self.contents['testsToRun'] = self.contents['tests']
if self.contents['loop'] == 1:
self.contents['testsToRun'] = [self.contents['tests'][-1]]
self.contents['loop'] += 1
return self.contents['testsToRun']
# Initialize the contents dict.
if status:
totalTests = len(self.contents['tests'])
self.contents['start'] = 0
self.contents['end'] = totalTests - 2
mid = (self.contents['start'] + self.contents['end']) / 2
if 'result' in self.contents:
if self.contents['result'] == "PASS":
self.contents['end'] = mid
elif self.contents['result'] == "FAIL":
self.contents['start'] = mid + 1
mid = (self.contents['start'] + self.contents['end']) / 2
start = mid + 1
end = self.contents['end'] + 1
self.contents['testsToRun'] = self.contents['tests'][start:end]
if not self.contents['testsToRun']:
self.contents['testsToRun'].append(self.contents['tests'][mid])
self.contents['testsToRun'].append(self.contents['tests'][-1])
self.contents['loop'] += 1
return self.contents['testsToRun']
def summarize_chunk(self, options):
"This method is used summarize the results after the list of tests is run."
if options.bisectChunk == "default":
# if no expectedError that means all the tests have successfully
# passed.
if len(self.expectedError) == 0:
return -1
options.bisectChunk = self.expectedError.keys()[0]
self.summary.append(
"\tFound Error in test: %s" %
options.bisectChunk)
return 0
# If options.bisectChunk is not in self.result then we need to move to
# the next run.
if options.bisectChunk not in self.result:
return -1
self.summary.append("\tPass %d:" % self.contents['loop'])
if len(self.contents['testsToRun']) > 1:
self.summary.append(
"\t\t%d test files(start,end,failing). [%s, %s, %s]" % (len(
self.contents['testsToRun']),
self.contents['testsToRun'][0],
self.contents['testsToRun'][
-2],
self.contents['testsToRun'][
-1]))
else:
self.summary.append(
"\t\t1 test file [%s]" %
self.contents['testsToRun'][0])
return self.check_for_intermittent(options)
if self.result[options.bisectChunk] == "PASS":
self.summary.append("\t\tno failures found.")
if self.contents['loop'] == 1:
status = -1
else:
self.contents['result'] = "PASS"
status = 0
elif self.result[options.bisectChunk] == "FAIL":
if 'expectedError' not in self.contents:
self.summary.append("\t\t%s failed." %
self.contents['testsToRun'][-1])
self.contents['expectedError'] = self.expectedError[
options.bisectChunk]
status = 0
elif self.expectedError[options.bisectChunk] == self.contents['expectedError']:
self.summary.append(
"\t\t%s failed with expected error." % self.contents['testsToRun'][-1])
self.contents['result'] = "FAIL"
status = 0
# This code checks for test-bleedthrough. Should work for any
# algorithm.
numberOfTests = len(self.contents['testsToRun'])
if numberOfTests < 3:
# This means that only 2 tests are run. Since the last test
# is the failing test itself therefore the bleedthrough
# test is the first test
self.summary.append(
"TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the root cause for many of the above failures" %
self.contents['testsToRun'][0])
status = -1
else:
self.summary.append(
"\t\t%s failed with different error." % self.contents['testsToRun'][-1])
status = -1
return status
def check_for_intermittent(self, options):
"This method is used to check whether a test is an intermittent."
if self.result[options.bisectChunk] == "PASS":
self.summary.append(
"\t\tThe test %s passed." %
self.contents['testsToRun'][0])
if self.repeat > 0:
# loop is set to 1 to again run the single test.
self.contents['loop'] = 1
self.repeat -= 1
return 0
else:
if self.failcount > 0:
# -1 is being returned as the test is intermittent, so no need to bisect further.
return -1
# If the test does not fail even once, then proceed to next chunk for bisection.
# loop is set to 2 to proceed on bisection.
self.contents['loop'] = 2
return 1
elif self.result[options.bisectChunk] == "FAIL":
self.summary.append(
"\t\tThe test %s failed." %
self.contents['testsToRun'][0])
self.failcount += 1
self.contents['loop'] = 1
self.repeat -= 1
# self.max_failures is the maximum number of times a test is allowed
# to fail to be called an intermittent. If a test fails more than
# limit set, it is a perma-fail.
if self.failcount < self.max_failures:
if self.repeat == 0:
# -1 is being returned as the test is intermittent, so no need to bisect further.
return -1
return 0
else:
self.summary.append(
"TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the root cause for many of the above failures" %
self.contents['testsToRun'][0])
return -1
def print_summary(self):
"This method is used to print the recorded summary."
print "Bisection summary:"
for line in self.summary:
print line