From 0b6e73324a0eacfac5506e623e2f7552d5bdfced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Pag=C3=A8s?= Date: Sat, 21 Nov 2015 10:55:27 +0100 Subject: [PATCH] Bug 1225553 - Should validate that PERFHERDER_DATA works against the perfherder data schema in treeherder. r=jmaher --- .../mozharness/mozilla/testing/talos.py | 38 ++++++-- .../performance-artifact.json | 86 +++++++++++++++++++ 2 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 testing/talos/treeherder-schemas/performance-artifact.json diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py index d0cb517c8dd..27f6af12871 100755 --- a/testing/mozharness/mozharness/mozilla/testing/talos.py +++ b/testing/mozharness/mozharness/mozilla/testing/talos.py @@ -12,6 +12,7 @@ import os import pprint import copy import re +import json from mozharness.base.config import parse_config_file from mozharness.base.errors import PythonErrorList @@ -47,7 +48,7 @@ class TalosOutputParser(OutputParser): def __init__(self, **kwargs): super(TalosOutputParser, self).__init__(**kwargs) self.minidump_output = None - self.num_times_found_perf_data = 0 + self.found_perf_data = [] def update_worst_log_and_tbpl_levels(self, log_level, tbpl_level): self.worst_log_level = self.worst_level(log_level, @@ -66,8 +67,9 @@ class TalosOutputParser(OutputParser): if m: self.minidump_output = (m.group(1), m.group(2), m.group(3)) - if self.RE_PERF_DATA.match(line): - self.num_times_found_perf_data += 1 + m = self.RE_PERF_DATA.match(line) + if m: + self.found_perf_data.append(m.group(1)) # now let's check if buildbot should retry harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex'] @@ -316,6 +318,30 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin): requirements=[os.path.join(self.talos_path, 'requirements.txt')] ) + # install jsonschema for perfherder validation + self.install_module(module="jsonschema") + + def _validate_treeherder_data(self, parser): + # late import is required, because install is done in create_virtualenv + import jsonschema + + if len(parser.found_perf_data) != 1: + self.critical("PERFHERDER_DATA was seen %d times, expected 1." + % len(parser.found_perf_data)) + parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING) + return + + schema_path = os.path.join(self.talos_path, 'treeherder-schemas', + 'performance-artifact.json') + self.info("Validating PERFHERDER_DATA against %s" % schema_path) + try: + with open(schema_path) as f: + schema = json.load(f) + data = json.loads(parser.found_perf_data[0]) + jsonschema.validate(data, schema) + except: + self.exception("Error while validating PERFHERDER_DATA") + parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING) def run_tests(self, args=None, **kw): """run Talos tests""" @@ -368,10 +394,8 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin): tbpl_level = TBPL_RETRY parser.update_worst_log_and_tbpl_levels(log_level, tbpl_level) - elif parser.num_times_found_perf_data != 1: - self.critical("PERFHERDER_DATA was seen %d times, expected 1." - % parser.num_times_found_perf_data) - parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING) + else: + self._validate_treeherder_data(parser) self.buildbot_status(parser.worst_tbpl_status, level=parser.worst_log_level) diff --git a/testing/talos/treeherder-schemas/performance-artifact.json b/testing/talos/treeherder-schemas/performance-artifact.json new file mode 100644 index 00000000000..85f624a2d26 --- /dev/null +++ b/testing/talos/treeherder-schemas/performance-artifact.json @@ -0,0 +1,86 @@ +{ + "definitions": { + "framework_schema": { + "properties": { + "name": { + "title": "Framework name", + "type": "string" + } + }, + "type": "object" + }, + "subtest_schema": { + "properties": { + "name": { + "title": "Subtest name", + "type": "string" + }, + "value": { + "description": "Summary value for subtest", + "title": "Subtest value", + "type": "number" + }, + "lowerIsBetter": { + "description": "Whether lower values are better for subtest", + "title": "Lower is better", + "type": "boolean" + } + }, + "required": [ + "name", + "value" + ], + "type": "object" + }, + "suite_schema": { + "properties": { + "name": { + "title": "Suite name", + "type": "string" + }, + "subtests": { + "items": { + "$ref": "#/definitions/subtest_schema" + }, + "title": "Subtests", + "type": "array" + }, + "value": { + "title": "Suite value", + "type": "number" + }, + "lowerIsBetter": { + "description": "Whether lower values are better for suite", + "title": "Lower is better", + "type": "boolean" + } + }, + "required": [ + "name", + "subtests" + ], + "type": "object" + } + }, + "description": "Structure for submitting performance data as part of a job", + "id": "https://treeherder.mozilla.org/schemas/v1/performance-artifact.json#", + "properties": { + "framework": { + "$ref": "#/definitions/framework_schema" + }, + "suites": { + "description": "List of suite-level data submitted as part of this structure", + "items": { + "$ref": "#/definitions/suite_schema" + }, + "title": "Performance suites", + "type": "array" + } + }, + "required": [ + "framework", + "suites" + ], + "title": "Perfherder Schema", + "type": "object" +}