# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""output formats for Talos"""
# NOTE: we have a circular dependency with output.py when we import results
import simplejson
as json
from mozlog
import get_proxy_logger
from talos
import filter, utils
LOG = get_proxy_logger()
class Output(object):
"""abstract base class for Talos output"""
@classmethod
def check(cls, urls):
"""check to ensure that the urls are valid"""
def __init__(self, results, tsresult_class):
"""
- results : TalosResults instance
- tsresult_class : Results
class
"""
self.results = results
self.tsresult_class = tsresult_class
def __call__(self):
suites = []
test_results = {
"application": {
"name": self.results.results[0].browser_name.lower(),
"version": self.results.results[0].browser_version,
},
"framework": {
"name": self.results.results[0].framework,
},
"suites": suites,
}
for test
in self.results.results:
# serialize test results
tsresult =
None
if not test.using_xperf:
subtests = []
suite = {
"name": test.name(),
"extraOptions": self.results.extra_options
or [],
"subtests": subtests,
"shouldAlert": test.test_config.get(
"suite_should_alert",
True),
}
suites.append(suite)
vals = []
replicates = {}
# TODO: counters!!!! we don't have any, but they suffer the same
for result
in test.results:
# XXX this will not work for manifests which list
# the same page name twice. It also ignores cycles
for page, val
in result.raw_values():
if page ==
"NULL":
page = test.name()
if tsresult
is None:
tsresult = r = self.tsresult_class()
r.results = [
{
"index": 0,
"page": test.name(),
"runs": val}
]
else:
r = tsresult.results[0]
if r[
"page"] == test.name():
r[
"runs"].extend(val)
replicates.setdefault(page, []).extend(val)
tresults = [tsresult]
if tsresult
else test.results
# Merge results for the same page when using cycle > 1
merged_results = {}
for result
in tresults:
results = []
for r
in result.results:
page = r[
"page"]
if page
in merged_results:
merged_results[page][
"runs"].extend(r[
"runs"])
else:
merged_results[page] = r
results.append(r)
# override the list of page results for each run
result.results = results
for result
in tresults:
filtered_results = result.values(
suite[
"name"], test.test_config[
"filters"]
)
vals.extend([[i[
"value"], j]
for i, j
in filtered_results])
subtest_index = 0
for val, page
in filtered_results:
if page ==
"NULL":
# no real subtests
page = test.name()
subtest = {
"name": page,
"value": val[
"filtered"],
"replicates": replicates[page],
}
# if results are from a comparison test i.e. perf-reftest, it will also
# contain replicates for 'base' and 'reference'; we wish to keep those
# to reference; actual results were calculated as the difference of those
base_runs = result.results[subtest_index].get(
"base_runs",
None)
ref_runs = result.results[subtest_index].get(
"ref_runs",
None)
if base_runs
and ref_runs:
subtest[
"base_replicates"] = base_runs
subtest[
"ref_replicates"] = ref_runs
subtests.append(subtest)
subtest_index += 1
if test.test_config.get(
"lower_is_better")
is not None:
subtest[
"lowerIsBetter"] = test.test_config[
"lower_is_better"
]
if test.test_config.get(
"alert_threshold")
is not None:
subtest[
"alertThreshold"] = test.test_config[
"alert_threshold"
]
if test.test_config.get(
"subtest_alerts")
is not None:
subtest[
"shouldAlert"] = test.test_config[
"subtest_alerts"]
if test.test_config.get(
"alert_threshold")
is not None:
subtest[
"alertThreshold"] = test.test_config[
"alert_threshold"
]
if test.test_config.get(
"unit"):
subtest[
"unit"] = test.test_config[
"unit"]
# if there is only one subtest, carry alerting setting from the suite
if len(subtests) == 1:
subtests[0][
"shouldAlert"] = suite[
"shouldAlert"]
# if there is more than one subtest, calculate a summary result
elif len(subtests) > 1:
suite[
"value"] = self.construct_results(vals, testname=test.name())
if test.test_config.get(
"lower_is_better")
is not None:
suite[
"lowerIsBetter"] = test.test_config[
"lower_is_better"]
if test.test_config.get(
"alert_threshold")
is not None:
suite[
"alertThreshold"] = test.test_config[
"alert_threshold"]
# counters results_aux data
counter_subtests = []
for cd
in test.all_counter_results:
for name, vals
in cd.items():
# We want to add the xperf data as talos_counters
# exclude counters whose values are tuples (bad for
# graphserver)
if len(vals) > 0
and isinstance(vals[0], list):
continue
# mainthread IO is a list of filenames and accesses, we do
# not report this as a counter
if "mainthreadio" in name:
continue
subtest = {
"name": name,
"value": 0.0,
}
counter_subtests.append(subtest)
if test.using_xperf:
if len(vals) > 0:
subtest[
"value"] = vals[0]
else:
# calculate mean value
if len(vals) > 0:
varray = [float(v)
for v
in vals]
subtest[
"value"] = filter.mean(varray)
if counter_subtests:
suites.append(
{
"name": test.name(),
"extraOptions": self.results.extra_options
or [],
"subtests": counter_subtests,
"shouldAlert": test.test_config.get(
"suite_should_alert",
True),
}
)
return test_results
def output(self, results, results_url):
"""output to the a file if results_url starts with file://
- results : json instance
- results_url : file:// URL
"""
# parse the results url
results_url_split = utils.urlsplit(results_url)
results_scheme, results_server, results_path, _, _ = results_url_split
if results_scheme
in (
"http",
"https"):
self.post(results, results_server, results_path, results_scheme)
elif results_scheme ==
"file":
with open(results_path,
"w")
as f:
for result
in results:
f.write(
"%s\n" % result)
else:
raise NotImplementedError(
"%s: %s - only http://, https://, and file:// supported"
% (self.__class__.__name__, results_url)
)
# This is the output that treeherder expects to find when parsing the
# log file
if "gecko-profile" in self.results.extra_options:
LOG.info(
"gecko-profile enabled")
for suite
in results[
"suites"]:
suite[
"shouldAlert"] =
False
for subtest
in suite[
"subtests"]:
subtest[
"shouldAlert"] =
False
LOG.info(
"PERFHERDER_DATA: %s" % json.dumps(results, ignore_nan=
True))
if results_scheme
in (
"file"):
json.dump(
results,
open(results_path,
"w"),
indent=2,
sort_keys=
True,
ignore_nan=
True,
)
def post(self, results, server, path, scheme):
raise NotImplementedError(
"Abstract base class")
@classmethod
def shortName(cls, name):
"""short name for counters"""
names = {
"% Processor Time":
"%cpu",
"XRes":
"xres"}
return names.get(name, name)
@classmethod
def isMemoryMetric(cls, resultName):
"""returns if the result is a memory metric"""
memory_metric = [
"xres"]
# measured in bytes
return bool([i
for i
in memory_metric
if i
in resultName])
@classmethod
def v8_Metric(cls, val_list):
results = [i
for i, j
in val_list]
score = 100 * filter.geometric_mean(results)
return score
@classmethod
def JS_Metric(cls, val_list):
"""v8 benchmark score"""
results = [i
for i, j
in val_list]
return sum(results)
@classmethod
def benchmark_score(cls, val_list):
"""
benchmark_score: ares6/jetstream self reported
as 'geomean'
"""
results = [i
for i, j
in val_list
if j ==
"geomean"]
return filter.mean(results)
@classmethod
def stylebench_score(cls, val_list):
"""
stylebench_score:
https://bug-172968-attachments.webkit.org/attachment.cgi?id=319888
"""
correctionFactor = 3
results = [i
for i, j
in val_list]
# stylebench has 5 tests, each of these are made of up 5 subtests
#
# * Adding classes.
# * Removing classes.
# * Mutating attributes.
# * Adding leaf elements.
# * Removing leaf elements.
#
# which are made of two subtests each (sync/async) and repeated 5 times
# each, thus, the list here looks like:
#
# [Test name/Adding classes - 0/ Sync; <x>]
# [Test name/Adding classes - 0/ Async; <y>]
# [Test name/Adding classes - 0; <x> + <y>]
# [Test name/Removing classes - 0/ Sync; <x>]
# [Test name/Removing classes - 0/ Async; <y>]
# [Test name/Removing classes - 0; <x> + <y>]
# ...
# [Test name/Adding classes - 1 / Sync; <x>]
# [Test name/Adding classes - 1 / Async; <y>]
# [Test name/Adding classes - 1 ; <x> + <y>]
# ...
# [Test name/Removing leaf elements - 4; <x> + <y>]
# [Test name; <sum>] <- This is what we want.
#
# So, 5 (subtests) *
# 5 (repetitions) *
# 3 (entries per repetition (sync/async/sum)) =
# 75 entries for test before the sum.
#
# We receive 76 entries per test, which ads up to 380. We want to use
# the 5 test entries, not the rest.
if len(results) != 380:
raise Exception(
"StyleBench requires 380 entries, found: %s instead" % len(results)
)
results = results[75::76]
# pylint --py3k W1619
score = 60 * 1000 / filter.geometric_mean(results) / correctionFactor
return score
@classmethod
def damp_score(cls, val_list):
"""
damp_score: damp
is only interested
in the value of subtests
and will
aggregate data
from several suites.
Use a hardcoded value
for the suite to avoid inconsistencies.
"""
return 100
def construct_results(self, vals, testname):
if testname.startswith(
"v8_7"):
return self.v8_Metric(vals)
elif testname.startswith(
"kraken"):
return self.JS_Metric(vals)
elif testname.startswith(
"ares6"):
return self.benchmark_score(vals)
elif testname.startswith(
"jetstream"):
return self.benchmark_score(vals)
elif testname.startswith(
"speedometer"):
return self.speedometer_score(vals)
elif testname.startswith(
"stylebench"):
return self.stylebench_score(vals)
elif testname.startswith(
"damp"):
return self.damp_score(vals)
elif len(vals) > 1:
return filter.geometric_mean([i
for i, j
in vals])
else:
return filter.mean([i
for i, j
in vals])
