You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
331 lines
14 KiB
331 lines
14 KiB
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
|
# Use of this source code is governed by a BSD-style license that can be |
|
# found in the LICENSE file. |
|
"""A module to generate experiments.""" |
|
|
|
from __future__ import print_function |
|
import os |
|
import re |
|
import socket |
|
|
|
from benchmark import Benchmark |
|
import config |
|
from experiment import Experiment |
|
from label import Label |
|
from label import MockLabel |
|
from results_cache import CacheConditions |
|
import test_flag |
|
import file_lock_machine |
|
|
|
# Users may want to run Telemetry tests either individually, or in |
|
# specified sets. Here we define sets of tests that users may want |
|
# to run together. |
|
|
|
telemetry_perfv2_tests = [ |
|
'dromaeo.domcoreattr', 'dromaeo.domcoremodify', 'dromaeo.domcorequery', |
|
'dromaeo.domcoretraverse', 'kraken', 'octane', 'robohornet_pro', 'sunspider' |
|
] |
|
|
|
telemetry_pagecycler_tests = [ |
|
'page_cycler_v2.intl_ar_fa_he', |
|
'page_cycler_v2.intl_es_fr_pt-BR', |
|
'page_cycler_v2.intl_hi_ru', |
|
'page_cycler_v2.intl_ja_zh', |
|
'page_cycler_v2.intl_ko_th_vi', |
|
# 'page_cycler_v2.morejs', |
|
# 'page_cycler_v2.moz', |
|
# 'page_cycler_v2.netsim.top_10', |
|
'page_cycler_v2.tough_layout_cases', |
|
'page_cycler_v2.typical_25' |
|
] |
|
|
|
telemetry_toolchain_old_perf_tests = [ |
|
'dromaeo.domcoremodify', 'page_cycler_v2.intl_es_fr_pt-BR', |
|
'page_cycler_v2.intl_hi_ru', 'page_cycler_v2.intl_ja_zh', |
|
'page_cycler_v2.intl_ko_th_vi', 'page_cycler_v2.netsim.top_10', |
|
'page_cycler_v2.typical_25', 'robohornet_pro', 'spaceport', |
|
'tab_switching.top_10' |
|
] |
|
telemetry_toolchain_perf_tests = [ |
|
'octane', |
|
'kraken', |
|
'speedometer', |
|
'dromaeo.domcoreattr', |
|
'dromaeo.domcoremodify', |
|
'smoothness.tough_webgl_cases', |
|
] |
|
graphics_perf_tests = [ |
|
'graphics_GLBench', |
|
'graphics_GLMark2', |
|
'graphics_SanAngeles', |
|
'graphics_WebGLAquarium', |
|
'graphics_WebGLPerformance', |
|
] |
|
telemetry_crosbolt_perf_tests = [ |
|
'octane', |
|
'kraken', |
|
'speedometer', |
|
'jetstream', |
|
'startup.cold.blank_page', |
|
'smoothness.top_25_smooth', |
|
] |
|
crosbolt_perf_tests = [ |
|
'graphics_WebGLAquarium', |
|
'video_PlaybackPerf.h264', |
|
'video_PlaybackPerf.vp9', |
|
'video_WebRtcPerf', |
|
'BootPerfServerCrosPerf', |
|
'power_Resume', |
|
'video_PlaybackPerf.h264', |
|
'build_RootFilesystemSize', |
|
# 'cheets_AntutuTest', |
|
# 'cheets_PerfBootServer', |
|
# 'cheets_CandyCrushTest', |
|
# 'cheets_LinpackTest', |
|
] |
|
|
|
|
|
class ExperimentFactory(object): |
|
"""Factory class for building an Experiment, given an ExperimentFile as input. |
|
|
|
This factory is currently hardcoded to produce an experiment for running |
|
ChromeOS benchmarks, but the idea is that in the future, other types |
|
of experiments could be produced. |
|
""" |
|
|
|
def AppendBenchmarkSet(self, benchmarks, benchmark_list, test_args, |
|
iterations, rm_chroot_tmp, perf_args, suite, |
|
show_all_results, retries, run_local): |
|
"""Add all the tests in a set to the benchmarks list.""" |
|
for test_name in benchmark_list: |
|
telemetry_benchmark = Benchmark(test_name, test_name, test_args, |
|
iterations, rm_chroot_tmp, perf_args, |
|
suite, show_all_results, retries, |
|
run_local) |
|
benchmarks.append(telemetry_benchmark) |
|
|
|
def GetExperiment(self, experiment_file, working_directory, log_dir): |
|
"""Construct an experiment from an experiment file.""" |
|
global_settings = experiment_file.GetGlobalSettings() |
|
experiment_name = global_settings.GetField('name') |
|
board = global_settings.GetField('board') |
|
remote = global_settings.GetField('remote') |
|
# This is used to remove the ",' from the remote if user |
|
# add them to the remote string. |
|
new_remote = [] |
|
if remote: |
|
for i in remote: |
|
c = re.sub('["\']', '', i) |
|
new_remote.append(c) |
|
remote = new_remote |
|
chromeos_root = global_settings.GetField('chromeos_root') |
|
rm_chroot_tmp = global_settings.GetField('rm_chroot_tmp') |
|
perf_args = global_settings.GetField('perf_args') |
|
acquire_timeout = global_settings.GetField('acquire_timeout') |
|
cache_dir = global_settings.GetField('cache_dir') |
|
cache_only = global_settings.GetField('cache_only') |
|
config.AddConfig('no_email', global_settings.GetField('no_email')) |
|
share_cache = global_settings.GetField('share_cache') |
|
results_dir = global_settings.GetField('results_dir') |
|
use_file_locks = global_settings.GetField('use_file_locks') |
|
locks_dir = global_settings.GetField('locks_dir') |
|
# If we pass a blank locks_dir to the Experiment, it will use the AFE server |
|
# lock mechanism. So if the user specified use_file_locks, but did not |
|
# specify a locks dir, set the locks dir to the default locks dir in |
|
# file_lock_machine. |
|
if use_file_locks and not locks_dir: |
|
locks_dir = file_lock_machine.Machine.LOCKS_DIR |
|
chrome_src = global_settings.GetField('chrome_src') |
|
show_all_results = global_settings.GetField('show_all_results') |
|
log_level = global_settings.GetField('logging_level') |
|
if log_level not in ('quiet', 'average', 'verbose'): |
|
log_level = 'verbose' |
|
# Default cache hit conditions. The image checksum in the cache and the |
|
# computed checksum of the image must match. Also a cache file must exist. |
|
cache_conditions = [ |
|
CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH |
|
] |
|
if global_settings.GetField('rerun_if_failed'): |
|
cache_conditions.append(CacheConditions.RUN_SUCCEEDED) |
|
if global_settings.GetField('rerun'): |
|
cache_conditions.append(CacheConditions.FALSE) |
|
if global_settings.GetField('same_machine'): |
|
cache_conditions.append(CacheConditions.SAME_MACHINE_MATCH) |
|
if global_settings.GetField('same_specs'): |
|
cache_conditions.append(CacheConditions.MACHINES_MATCH) |
|
|
|
# Construct benchmarks. |
|
# Some fields are common with global settings. The values are |
|
# inherited and/or merged with the global settings values. |
|
benchmarks = [] |
|
all_benchmark_settings = experiment_file.GetSettings('benchmark') |
|
for benchmark_settings in all_benchmark_settings: |
|
benchmark_name = benchmark_settings.name |
|
test_name = benchmark_settings.GetField('test_name') |
|
if not test_name: |
|
test_name = benchmark_name |
|
test_args = benchmark_settings.GetField('test_args') |
|
iterations = benchmark_settings.GetField('iterations') |
|
suite = benchmark_settings.GetField('suite') |
|
retries = benchmark_settings.GetField('retries') |
|
run_local = benchmark_settings.GetField('run_local') |
|
|
|
if suite == 'telemetry_Crosperf': |
|
if test_name == 'all_perfv2': |
|
self.AppendBenchmarkSet(benchmarks, telemetry_perfv2_tests, test_args, |
|
iterations, rm_chroot_tmp, perf_args, suite, |
|
show_all_results, retries, run_local) |
|
elif test_name == 'all_pagecyclers': |
|
self.AppendBenchmarkSet(benchmarks, telemetry_pagecycler_tests, |
|
test_args, iterations, rm_chroot_tmp, |
|
perf_args, suite, show_all_results, retries, |
|
run_local) |
|
elif test_name == 'all_toolchain_perf': |
|
self.AppendBenchmarkSet(benchmarks, telemetry_toolchain_perf_tests, |
|
test_args, iterations, rm_chroot_tmp, |
|
perf_args, suite, show_all_results, retries, |
|
run_local) |
|
# Add non-telemetry toolchain-perf benchmarks: |
|
benchmarks.append( |
|
Benchmark( |
|
'graphics_WebGLAquarium', |
|
'graphics_WebGLAquarium', |
|
'', |
|
iterations, |
|
rm_chroot_tmp, |
|
perf_args, |
|
'', |
|
show_all_results, |
|
retries, |
|
run_local=False)) |
|
elif test_name == 'all_toolchain_perf_old': |
|
self.AppendBenchmarkSet(benchmarks, |
|
telemetry_toolchain_old_perf_tests, test_args, |
|
iterations, rm_chroot_tmp, perf_args, suite, |
|
show_all_results, retries, run_local) |
|
else: |
|
benchmark = Benchmark(test_name, test_name, test_args, iterations, |
|
rm_chroot_tmp, perf_args, suite, |
|
show_all_results, retries, run_local) |
|
benchmarks.append(benchmark) |
|
else: |
|
if test_name == 'all_graphics_perf': |
|
self.AppendBenchmarkSet(benchmarks, |
|
graphics_perf_tests, '', |
|
iterations, rm_chroot_tmp, perf_args, '', |
|
show_all_results, retries, run_local=False) |
|
elif test_name == 'all_crosbolt_perf': |
|
self.AppendBenchmarkSet(benchmarks, |
|
telemetry_crosbolt_perf_tests, test_args, |
|
iterations, rm_chroot_tmp, perf_args, |
|
'telemetry_Crosperf', show_all_results, |
|
retries, run_local) |
|
self.AppendBenchmarkSet(benchmarks, |
|
crosbolt_perf_tests, '', |
|
iterations, rm_chroot_tmp, perf_args, '', |
|
show_all_results, retries, run_local=False) |
|
else: |
|
# Add the single benchmark. |
|
benchmark = Benchmark( |
|
benchmark_name, |
|
test_name, |
|
test_args, |
|
iterations, |
|
rm_chroot_tmp, |
|
perf_args, |
|
suite, |
|
show_all_results, |
|
retries, |
|
run_local=False) |
|
benchmarks.append(benchmark) |
|
|
|
if not benchmarks: |
|
raise RuntimeError('No benchmarks specified') |
|
|
|
# Construct labels. |
|
# Some fields are common with global settings. The values are |
|
# inherited and/or merged with the global settings values. |
|
labels = [] |
|
all_label_settings = experiment_file.GetSettings('label') |
|
all_remote = list(remote) |
|
for label_settings in all_label_settings: |
|
label_name = label_settings.name |
|
image = label_settings.GetField('chromeos_image') |
|
autotest_path = label_settings.GetField('autotest_path') |
|
chromeos_root = label_settings.GetField('chromeos_root') |
|
my_remote = label_settings.GetField('remote') |
|
compiler = label_settings.GetField('compiler') |
|
new_remote = [] |
|
if my_remote: |
|
for i in my_remote: |
|
c = re.sub('["\']', '', i) |
|
new_remote.append(c) |
|
my_remote = new_remote |
|
if image == '': |
|
build = label_settings.GetField('build') |
|
if len(build) == 0: |
|
raise RuntimeError("Can not have empty 'build' field!") |
|
image, autotest_path = label_settings.GetXbuddyPath(build, |
|
autotest_path, |
|
board, |
|
chromeos_root, |
|
log_level) |
|
|
|
cache_dir = label_settings.GetField('cache_dir') |
|
chrome_src = label_settings.GetField('chrome_src') |
|
|
|
# TODO(yunlian): We should consolidate code in machine_manager.py |
|
# to derermine whether we are running from within google or not |
|
if ('corp.google.com' in socket.gethostname() and |
|
(not my_remote or my_remote == remote and |
|
global_settings.GetField('board') != board)): |
|
my_remote = self.GetDefaultRemotes(board) |
|
if global_settings.GetField('same_machine') and len(my_remote) > 1: |
|
raise RuntimeError('Only one remote is allowed when same_machine ' |
|
'is turned on') |
|
all_remote += my_remote |
|
image_args = label_settings.GetField('image_args') |
|
if test_flag.GetTestMode(): |
|
# pylint: disable=too-many-function-args |
|
label = MockLabel(label_name, image, autotest_path, chromeos_root, |
|
board, my_remote, image_args, cache_dir, cache_only, |
|
log_level, compiler, chrome_src) |
|
else: |
|
label = Label(label_name, image, autotest_path, chromeos_root, board, |
|
my_remote, image_args, cache_dir, cache_only, log_level, |
|
compiler, chrome_src) |
|
labels.append(label) |
|
|
|
if not labels: |
|
raise RuntimeError('No labels specified') |
|
|
|
email = global_settings.GetField('email') |
|
all_remote += list(set(my_remote)) |
|
all_remote = list(set(all_remote)) |
|
experiment = Experiment(experiment_name, all_remote, working_directory, |
|
chromeos_root, cache_conditions, labels, benchmarks, |
|
experiment_file.Canonicalize(), email, |
|
acquire_timeout, log_dir, log_level, share_cache, |
|
results_dir, locks_dir) |
|
|
|
return experiment |
|
|
|
def GetDefaultRemotes(self, board): |
|
default_remotes_file = os.path.join( |
|
os.path.dirname(__file__), 'default_remotes') |
|
try: |
|
with open(default_remotes_file) as f: |
|
for line in f: |
|
key, v = line.split(':') |
|
if key.strip() == board: |
|
remotes = v.strip().split() |
|
if remotes: |
|
return remotes |
|
else: |
|
raise RuntimeError('There is no remote for {0}'.format(board)) |
|
except IOError: |
|
# TODO: rethrow instead of throwing different exception. |
|
raise RuntimeError('IOError while reading file {0}' |
|
.format(default_remotes_file)) |
|
else: |
|
raise RuntimeError('There is not remote for {0}'.format(board))
|
|
|