From 0dbaa8b3de8d80c0b9d3ea69d1aecd4fa899d25c Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 21 May 2014 16:36:38 +0100 Subject: [PATCH] Bug 1011434 - Add a framework for structured logging command line scripts and a script for printing unstable tests, r=ahal --- .../mozlog/structured/scripts/__init__.py | 26 +++++ .../mozlog/structured/scripts/unstable.py | 109 ++++++++++++++++++ testing/mozbase/mozlog/setup.py | 6 +- 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 testing/mozbase/mozlog/mozlog/structured/scripts/__init__.py create mode 100644 testing/mozbase/mozlog/mozlog/structured/scripts/unstable.py diff --git a/testing/mozbase/mozlog/mozlog/structured/scripts/__init__.py b/testing/mozbase/mozlog/mozlog/structured/scripts/__init__.py new file mode 100644 index 00000000000..978033d9264 --- /dev/null +++ b/testing/mozbase/mozlog/mozlog/structured/scripts/__init__.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +import argparse +import unstable + +def get_parser(): + parser = argparse.ArgumentParser("structlog", + description="Tools for dealing with structured logs") + + commands = {"unstable": (unstable.get_parser, unstable.main)} + + sub_parser = parser.add_subparsers(title='Subcommands') + + for command, (parser_func, main_func) in commands.iteritems(): + parent = parser_func(False) + command_parser = sub_parser.add_parser(command, + description=parent.description, + parents=[parent]) + command_parser.set_defaults(func=main_func) + + return parser + +def main(): + parser = get_parser() + args = parser.parse_args() + args.func(**vars(args)) diff --git a/testing/mozbase/mozlog/mozlog/structured/scripts/unstable.py b/testing/mozbase/mozlog/mozlog/structured/scripts/unstable.py new file mode 100644 index 00000000000..1efd98c44bd --- /dev/null +++ b/testing/mozbase/mozlog/mozlog/structured/scripts/unstable.py @@ -0,0 +1,109 @@ +import sys +import argparse +from collections import defaultdict +import json + +from mozlog.structured import reader + +class StatusHandler(reader.LogHandler): + def __init__(self): + self.run_info = None + self.statuses = defaultdict(lambda:defaultdict(lambda:defaultdict(lambda: defaultdict(int)))) + + def test_id(self, test): + if type(test) in (str, unicode): + return test + else: + return tuple(test) + + def suite_start(self, item): + self.run_info = tuple(sorted(item.get("run_info", {}).items())) + + def test_status(self, item): + self.statuses[self.run_info][self.test_id(item["test"])][item["subtest"]][item["status"]] += 1 + + def test_end(self, item): + self.statuses[self.run_info][self.test_id(item["test"])][None][item["status"]] += 1 + + def suite_end(self, item): + self.run_info = None + +def get_statuses(filenames): + handler = StatusHandler() + + for filename in filenames: + with open(filename) as f: + reader.handle_log(reader.read(f), handler) + + return handler.statuses + +def _filter(results_cmp): + def inner(statuses): + rv = defaultdict(lambda:defaultdict(dict)) + + for run_info, tests in statuses.iteritems(): + for test, subtests in tests.iteritems(): + for name, results in subtests.iteritems(): + if results_cmp(results): + rv[run_info][test][name] = results + + return rv + return inner + +filter_unstable = _filter(lambda x: len(x) > 1) +filter_stable = _filter(lambda x: len(x) == 1) + +def group_results(data): + rv = defaultdict(lambda: defaultdict(lambda: defaultdict(int))) + + for run_info, tests in data.iteritems(): + for test, subtests in tests.iteritems(): + for name, results in subtests.iteritems(): + for status, number in results.iteritems(): + rv[test][name][status] += number + return rv + +def print_results(data): + for run_info, tests in data.iteritems(): + run_str = " ".join("%s:%s" % (k,v) for k,v in run_info) if run_info else "No Run Info" + print run_str + print "=" * len(run_str) + print_run(tests) + +def print_run(tests): + for test, subtests in sorted(tests.items()): + print "\n" + str(test) + print "-" * len(test) + for name, results in subtests.iteritems(): + print "[%s]: %s" % (name if name is not None else "", + " ".join("%s (%i)" % (k,v) for k,v in results.iteritems())) + +def get_parser(add_help=True): + parser = argparse.ArgumentParser("unstable", + description="List tests that don't give consistent results from one or more runs.", add_help=add_help) + parser.add_argument("--json", action="store_true", default=False, + help="Output in JSON format") + parser.add_argument("--group", action="store_true", default=False, + help="Group results from different run types") + parser.add_argument("log_file", nargs="+", + help="Log files to read") + return parser + +def main(**kwargs): + unstable = filter_unstable(get_statuses(kwargs["log_file"])) + if kwargs["group"]: + unstable = group_results(unstable) + + if kwargs["json"]: + print json.dumps(unstable) + else: + if not kwargs["group"]: + print_results(unstable) + else: + print_run(unstable) + +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + kwargs = vars(args) + main(**kwargs) diff --git a/testing/mozbase/mozlog/setup.py b/testing/mozbase/mozlog/setup.py index 267ce56b1b3..520fd178491 100644 --- a/testing/mozbase/mozlog/setup.py +++ b/testing/mozbase/mozlog/setup.py @@ -25,5 +25,9 @@ setup(name=PACKAGE_NAME, 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)', 'Operating System :: OS Independent', 'Topic :: Software Development :: Libraries :: Python Modules', - ] + ], + entry_points={ + "console_scripts": [ + "structlog = mozlog.structured.scripts:main" + ]} )