You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
356 lines
12 KiB
356 lines
12 KiB
#!/usr/bin/env python2 |
|
# |
|
# Copyright 2016 The Chromium OS Authors. All rights reserved. |
|
# Use of this source code is governed by a BSD-style license that can be |
|
# found in the LICENSE file. |
|
"""Script for running nightly compiler tests on ChromeOS. |
|
|
|
This script launches a buildbot to build ChromeOS with the latest compiler on |
|
a particular board; then it finds and downloads the trybot image and the |
|
corresponding official image, and runs crosperf performance tests comparing |
|
the two. It then generates a report, emails it to the c-compiler-chrome, as |
|
well as copying the images into the seven-day reports directory. |
|
""" |
|
|
|
# Script to test different toolchains against ChromeOS benchmarks. |
|
|
|
from __future__ import print_function |
|
|
|
import argparse |
|
import datetime |
|
import os |
|
import re |
|
import sys |
|
import time |
|
|
|
from cros_utils import command_executer |
|
from cros_utils import logger |
|
|
|
from cros_utils import buildbot_utils |
|
|
|
# CL that updated GCC ebuilds to use 'next_gcc'. |
|
USE_NEXT_GCC_PATCH = '230260' |
|
|
|
# CL that uses LLVM to build the peppy image. |
|
USE_LLVM_PATCH = '295217' |
|
|
|
# CL that uses LLVM-Next to build the images (includes chrome). |
|
USE_LLVM_NEXT_PATCH = '424123' |
|
|
|
CROSTC_ROOT = '/usr/local/google/crostc' |
|
ROLE_ACCOUNT = 'mobiletc-prebuild' |
|
TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) |
|
MAIL_PROGRAM = '~/var/bin/mail-sheriff' |
|
PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') |
|
NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports') |
|
|
|
IMAGE_DIR = '{board}-{image_type}' |
|
IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}' |
|
IMAGE_FS = IMAGE_DIR + '/' + IMAGE_VERSION_STR |
|
TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}' |
|
PFQ_IMAGE_FS = IMAGE_FS + '-rc1' |
|
IMAGE_RE_GROUPS = { |
|
'board': r'(?P<board>\S+)', |
|
'image_type': r'(?P<image_type>\S+)', |
|
'chrome_version': r'(?P<chrome_version>R\d+)', |
|
'tip': r'(?P<tip>\d+)', |
|
'branch': r'(?P<branch>\d+)', |
|
'branch_branch': r'(?P<branch_branch>\d+)', |
|
'build_id': r'(?P<build_id>b\d+)' |
|
} |
|
TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS) |
|
|
|
|
|
class ToolchainComparator(object): |
|
"""Class for doing the nightly tests work.""" |
|
|
|
def __init__(self, |
|
board, |
|
remotes, |
|
chromeos_root, |
|
weekday, |
|
patches, |
|
noschedv2=False): |
|
self._board = board |
|
self._remotes = remotes |
|
self._chromeos_root = chromeos_root |
|
self._base_dir = os.getcwd() |
|
self._ce = command_executer.GetCommandExecuter() |
|
self._l = logger.GetLogger() |
|
self._build = '%s-release' % board |
|
self._patches = patches.split(',') |
|
self._patches_string = '_'.join(str(p) for p in self._patches) |
|
self._noschedv2 = noschedv2 |
|
|
|
if not weekday: |
|
self._weekday = time.strftime('%a') |
|
else: |
|
self._weekday = weekday |
|
timestamp = datetime.datetime.strftime(datetime.datetime.now(), |
|
'%Y-%m-%d_%H:%M:%S') |
|
self._reports_dir = os.path.join( |
|
NIGHTLY_TESTS_DIR, |
|
'%s.%s' % (timestamp, board),) |
|
|
|
def _GetVanillaImageName(self, trybot_image): |
|
"""Given a trybot artifact name, get latest vanilla image name. |
|
|
|
Args: |
|
trybot_image: artifact name such as |
|
'trybot-daisy-release/R40-6394.0.0-b1389' |
|
|
|
Returns: |
|
Latest official image name, e.g. 'daisy-release/R57-9089.0.0'. |
|
""" |
|
mo = re.search(TRYBOT_IMAGE_RE, trybot_image) |
|
assert mo |
|
dirname = IMAGE_DIR.replace('\\', '').format(**mo.groupdict()) |
|
version = buildbot_utils.GetGSContent(self._chromeos_root, |
|
dirname + '/LATEST-master') |
|
return dirname + '/' + version |
|
|
|
def _GetNonAFDOImageName(self, trybot_image): |
|
"""Given a trybot artifact name, get corresponding non-AFDO image name. |
|
|
|
We get the non-AFDO image from the PFQ builders. This image |
|
is not generated for all the boards and, the closest PFQ image |
|
was the one build for the previous ChromeOS version (the chrome |
|
used in the current version is the one validated in the previous |
|
version). |
|
The previous ChromeOS does not always exist either. So, we try |
|
a couple of versions before. |
|
|
|
Args: |
|
trybot_image: artifact name such as |
|
'trybot-daisy-release/R40-6394.0.0-b1389' |
|
|
|
Returns: |
|
Corresponding chrome PFQ image name, e.g. |
|
'daisy-chrome-pfq/R40-6393.0.0-rc1'. |
|
""" |
|
mo = re.search(TRYBOT_IMAGE_RE, trybot_image) |
|
assert mo |
|
image_dict = mo.groupdict() |
|
image_dict['image_type'] = 'chrome-pfq' |
|
for _ in xrange(2): |
|
image_dict['tip'] = str(int(image_dict['tip']) - 1) |
|
nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict) |
|
if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image): |
|
return nonafdo_image |
|
return '' |
|
|
|
def _FinishSetup(self): |
|
"""Make sure testing_rsa file is properly set up.""" |
|
# Fix protections on ssh key |
|
command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' |
|
'/chrome-src-internal/src/third_party/chromite/ssh_keys' |
|
'/testing_rsa') |
|
ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) |
|
if ret_val != 0: |
|
raise RuntimeError('chmod for testing_rsa failed') |
|
|
|
def _TestImages(self, trybot_image, vanilla_image, nonafdo_image): |
|
"""Create crosperf experiment file. |
|
|
|
Given the names of the trybot, vanilla and non-AFDO images, create the |
|
appropriate crosperf experiment file and launch crosperf on it. |
|
""" |
|
experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday) |
|
experiment_file_name = '%s_toolchain_experiment.txt' % self._board |
|
|
|
compiler_string = 'gcc' |
|
if USE_LLVM_NEXT_PATCH in self._patches_string: |
|
experiment_file_name = '%s_llvm_next_experiment.txt' % self._board |
|
compiler_string = 'llvm_next' |
|
elif USE_LLVM_PATCH in self._patches_string: |
|
experiment_file_name = '%s_llvm_experiment.txt' % self._board |
|
compiler_string = 'llvm' |
|
|
|
experiment_file = os.path.join(experiment_file_dir, experiment_file_name) |
|
experiment_header = """ |
|
board: %s |
|
remote: %s |
|
retries: 1 |
|
""" % (self._board, self._remotes) |
|
experiment_tests = """ |
|
benchmark: all_toolchain_perf { |
|
suite: telemetry_Crosperf |
|
iterations: 3 |
|
} |
|
|
|
benchmark: page_cycler_v2.typical_25 { |
|
suite: telemetry_Crosperf |
|
iterations: 2 |
|
run_local: False |
|
retries: 0 |
|
} |
|
""" |
|
|
|
with open(experiment_file, 'w') as f: |
|
f.write(experiment_header) |
|
f.write(experiment_tests) |
|
|
|
# Now add vanilla to test file. |
|
official_image = """ |
|
vanilla_image { |
|
chromeos_root: %s |
|
build: %s |
|
compiler: gcc |
|
} |
|
""" % (self._chromeos_root, vanilla_image) |
|
f.write(official_image) |
|
|
|
# Now add non-AFDO image to test file. |
|
if nonafdo_image: |
|
official_nonafdo_image = """ |
|
nonafdo_image { |
|
chromeos_root: %s |
|
build: %s |
|
compiler: gcc |
|
} |
|
""" % (self._chromeos_root, nonafdo_image) |
|
f.write(official_nonafdo_image) |
|
|
|
label_string = '%s_trybot_image' % compiler_string |
|
if USE_NEXT_GCC_PATCH in self._patches: |
|
label_string = 'gcc_next_trybot_image' |
|
|
|
# Reuse autotest files from vanilla image for trybot images |
|
autotest_files = os.path.join('/tmp', vanilla_image, 'autotest_files') |
|
experiment_image = """ |
|
%s { |
|
chromeos_root: %s |
|
build: %s |
|
autotest_path: %s |
|
compiler: %s |
|
} |
|
""" % (label_string, self._chromeos_root, trybot_image, |
|
autotest_files, compiler_string) |
|
f.write(experiment_image) |
|
|
|
crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf') |
|
noschedv2_opts = '--noschedv2' if self._noschedv2 else '' |
|
command = ('{crosperf} --no_email=True --results_dir={r_dir} ' |
|
'--json_report=True {noschedv2_opts} {exp_file}').format( |
|
crosperf=crosperf, |
|
r_dir=self._reports_dir, |
|
noschedv2_opts=noschedv2_opts, |
|
exp_file=experiment_file) |
|
|
|
ret = self._ce.RunCommand(command) |
|
if ret != 0: |
|
raise RuntimeError('Crosperf execution error!') |
|
else: |
|
# Copy json report to pending archives directory. |
|
command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) |
|
ret = self._ce.RunCommand(command) |
|
return |
|
|
|
def _SendEmail(self): |
|
"""Find email message generated by crosperf and send it.""" |
|
filename = os.path.join(self._reports_dir, 'msg_body.html') |
|
if (os.path.exists(filename) and |
|
os.path.exists(os.path.expanduser(MAIL_PROGRAM))): |
|
email_title = 'buildbot test results' |
|
if USE_LLVM_NEXT_PATCH in self._patches_string: |
|
email_title = 'buildbot llvm_next test results' |
|
elif USE_LLVM_PATCH in self._patches_string: |
|
email_title = 'buildbot llvm test results' |
|
command = ('cat %s | %s -s "%s, %s" -team -html' % |
|
(filename, MAIL_PROGRAM, email_title, self._board)) |
|
self._ce.RunCommand(command) |
|
|
|
def DoAll(self): |
|
"""Main function inside ToolchainComparator class. |
|
|
|
Launch trybot, get image names, create crosperf experiment file, run |
|
crosperf, and copy images into seven-day report directories. |
|
""" |
|
date_str = datetime.date.today() |
|
description = 'master_%s_%s_%s' % (self._patches_string, self._build, |
|
date_str) |
|
build_id, trybot_image = buildbot_utils.GetTrybotImage( |
|
self._chromeos_root, |
|
self._build, |
|
self._patches, |
|
description, |
|
other_flags=['--notests'], |
|
build_toolchain=True) |
|
|
|
print('trybot_url: \ |
|
https://uberchromegw.corp.google.com/i/chromiumos.tryserver/builders/release/builds/%s' |
|
% build_id) |
|
if len(trybot_image) == 0: |
|
self._l.LogError('Unable to find trybot_image for %s!' % description) |
|
return 1 |
|
|
|
vanilla_image = self._GetVanillaImageName(trybot_image) |
|
nonafdo_image = self._GetNonAFDOImageName(trybot_image) |
|
|
|
print('trybot_image: %s' % trybot_image) |
|
print('vanilla_image: %s' % vanilla_image) |
|
print('nonafdo_image: %s' % nonafdo_image) |
|
|
|
if os.getlogin() == ROLE_ACCOUNT: |
|
self._FinishSetup() |
|
|
|
self._TestImages(trybot_image, vanilla_image, nonafdo_image) |
|
self._SendEmail() |
|
return 0 |
|
|
|
|
|
def Main(argv): |
|
"""The main function.""" |
|
|
|
# Common initializations |
|
command_executer.InitCommandExecuter() |
|
parser = argparse.ArgumentParser() |
|
parser.add_argument( |
|
'--remote', dest='remote', help='Remote machines to run tests on.') |
|
parser.add_argument( |
|
'--board', dest='board', default='x86-zgb', help='The target board.') |
|
parser.add_argument( |
|
'--chromeos_root', |
|
dest='chromeos_root', |
|
help='The chromeos root from which to run tests.') |
|
parser.add_argument( |
|
'--weekday', |
|
default='', |
|
dest='weekday', |
|
help='The day of the week for which to run tests.') |
|
parser.add_argument( |
|
'--patch', |
|
dest='patches', |
|
help='The patches to use for the testing, ' |
|
"seprate the patch numbers with ',' " |
|
'for more than one patches.') |
|
parser.add_argument( |
|
'--noschedv2', |
|
dest='noschedv2', |
|
action='store_true', |
|
default=False, |
|
help='Pass --noschedv2 to crosperf.') |
|
|
|
options = parser.parse_args(argv[1:]) |
|
if not options.board: |
|
print('Please give a board.') |
|
return 1 |
|
if not options.remote: |
|
print('Please give at least one remote machine.') |
|
return 1 |
|
if not options.chromeos_root: |
|
print('Please specify the ChromeOS root directory.') |
|
return 1 |
|
if options.patches: |
|
patches = options.patches |
|
else: |
|
patches = USE_NEXT_GCC_PATCH |
|
|
|
fc = ToolchainComparator(options.board, options.remote, options.chromeos_root, |
|
options.weekday, patches, options.noschedv2) |
|
return fc.DoAll() |
|
|
|
|
|
if __name__ == '__main__': |
|
retval = Main(sys.argv) |
|
sys.exit(retval)
|
|
|