You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
385 lines
13 KiB
385 lines
13 KiB
# Copyright 2016 The Chromium OS Authors. All rights reserved. |
|
# Use of this source code is governed by a BSD-style license that can be |
|
# found in the LICENSE file. |
|
"""Utilities for launching and accessing ChromeOS buildbots.""" |
|
|
|
from __future__ import print_function |
|
|
|
import base64 |
|
import json |
|
import os |
|
import time |
|
import urllib2 |
|
|
|
# pylint: disable=no-name-in-module |
|
from oauth2client.service_account import ServiceAccountCredentials |
|
|
|
from cros_utils import command_executer |
|
from cros_utils import logger |
|
from cros_utils import buildbot_json |
|
|
|
INITIAL_SLEEP_TIME = 7200 # 2 hours; wait time before polling buildbot. |
|
SLEEP_TIME = 600 # 10 minutes; time between polling of buildbot. |
|
TIME_OUT = 28800 # Decide the build is dead or will never finish |
|
# after this time (8 hours). |
|
OK_STATUS = [ # List of result status values that are 'ok'. |
|
# This was obtained from: |
|
# https://chromium.googlesource.com/chromium/tools/build/+/ |
|
# master/third_party/buildbot_8_4p1/buildbot/status/results.py |
|
0, # "success" |
|
1, # "warnings" |
|
6, # "retry" |
|
] |
|
|
|
|
|
class BuildbotTimeout(Exception): |
|
"""Exception to throw when a buildbot operation timesout.""" |
|
pass |
|
|
|
|
|
def ParseReportLog(url, build): |
|
"""Scrape the trybot image name off the Reports log page. |
|
|
|
This takes the URL for a trybot Reports Stage web page, |
|
and a trybot build type, such as 'daisy-release'. It |
|
opens the web page and parses it looking for the trybot |
|
artifact name (e.g. something like |
|
'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the |
|
artifact name, if found. |
|
""" |
|
trybot_image = '' |
|
url += '/text' |
|
newurl = url.replace('uberchromegw', 'chromegw') |
|
webpage = urllib2.urlopen(newurl) |
|
data = webpage.read() |
|
lines = data.split('\n') |
|
for l in lines: |
|
if l.find('Artifacts') > 0 and l.find('trybot') > 0: |
|
trybot_name = 'trybot-%s' % build |
|
start_pos = l.find(trybot_name) |
|
end_pos = l.find('@https://storage') |
|
trybot_image = l[start_pos:end_pos] |
|
|
|
return trybot_image |
|
|
|
|
|
def GetBuildData(buildbot_queue, build_id): |
|
"""Find the Reports stage web page for a trybot build. |
|
|
|
This takes the name of a buildbot_queue, such as 'daisy-release' |
|
and a build id (the build number), and uses the json buildbot api to |
|
find the Reports stage web page for that build, if it exists. |
|
""" |
|
builder = buildbot_json.Buildbot( |
|
'http://chromegw/p/tryserver.chromiumos/').builders[buildbot_queue] |
|
build_data = builder.builds[build_id].data |
|
logs = build_data['logs'] |
|
for l in logs: |
|
fname = l[1] |
|
if 'steps/Report/' in fname: |
|
return fname |
|
|
|
return '' |
|
|
|
|
|
def FindBuildRecordFromLog(description, build_info): |
|
"""Find the right build record in the build logs. |
|
|
|
Get the first build record from build log with a reason field |
|
that matches 'description'. ('description' is a special tag we |
|
created when we launched the buildbot, so we could find it at this |
|
point.) |
|
""" |
|
for build_log in build_info: |
|
if description in build_log['reason']: |
|
return build_log |
|
return {} |
|
|
|
|
|
def GetBuildInfo(file_dir, waterfall_builder): |
|
"""Get all the build records for the trybot builds.""" |
|
|
|
builder = '' |
|
if waterfall_builder.endswith('-release'): |
|
builder = 'release' |
|
elif waterfall_builder.endswith('-gcc-toolchain'): |
|
builder = 'gcc_toolchain' |
|
elif waterfall_builder.endswith('-llvm-toolchain'): |
|
builder = 'llvm_toolchain' |
|
elif waterfall_builder.endswith('-llvm-next-toolchain'): |
|
builder = 'llvm_next_toolchain' |
|
|
|
sa_file = os.path.expanduser( |
|
os.path.join(file_dir, 'cros_utils', |
|
'chromeos-toolchain-credentials.json')) |
|
scopes = ['https://www.googleapis.com/auth/userinfo.email'] |
|
|
|
credentials = ServiceAccountCredentials.from_json_keyfile_name( |
|
sa_file, scopes=scopes) |
|
url = ( |
|
'https://luci-milo.appspot.com/prpc/milo.Buildbot/GetBuildbotBuildsJSON') |
|
|
|
# NOTE: If we want to get build logs for the main waterfall builders, the |
|
# 'master' field below should be 'chromeos' instead of 'chromiumos.tryserver'. |
|
# Builder would be 'amd64-gcc-toolchain' or 'arm-llvm-toolchain', etc. |
|
|
|
body = json.dumps({ |
|
'master': 'chromiumos.tryserver', |
|
'builder': builder, |
|
'include_current': True, |
|
'limit': 100 |
|
}) |
|
access_token = credentials.get_access_token() |
|
headers = { |
|
'Accept': 'application/json', |
|
'Content-Type': 'application/json', |
|
'Authorization': 'Bearer %s' % access_token.access_token |
|
} |
|
r = urllib2.Request(url, body, headers) |
|
u = urllib2.urlopen(r, timeout=60) |
|
u.read(4) |
|
o = json.load(u) |
|
data = [base64.b64decode(item['data']) for item in o['builds']] |
|
result = [] |
|
for d in data: |
|
tmp = json.loads(d) |
|
result.append(tmp) |
|
return result |
|
|
|
|
|
def FindArchiveImage(chromeos_root, build, build_id): |
|
"""Returns name of the trybot artifact for board/build_id.""" |
|
ce = command_executer.GetCommandExecuter() |
|
command = ('gsutil ls gs://chromeos-image-archive/trybot-%s/*b%s' |
|
'/chromiumos_test_image.tar.xz' % (build, build_id)) |
|
_, out, _ = ce.ChrootRunCommandWOutput( |
|
chromeos_root, command, print_to_console=False) |
|
# |
|
# If build_id is not unique, there may be multiple archive images |
|
# to choose from; sort them & pick the first (newest). |
|
# |
|
# If there are multiple archive images found, out will look something |
|
# like this: |
|
# |
|
# 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz |
|
# gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' |
|
# |
|
out = out.rstrip('\n') |
|
tmp_list = out.split('\n') |
|
# After stripping the final '\n' and splitting on any other '\n', we get |
|
# something like this: |
|
# tmp_list = [ 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz' , |
|
# 'gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' ] |
|
# |
|
# If we sort this in descending order, we should end up with the most |
|
# recent test image first, so that's what we do here. |
|
# |
|
if len(tmp_list) > 1: |
|
tmp_list = sorted(tmp_list, reverse=True) |
|
out = tmp_list[0] |
|
|
|
trybot_image = '' |
|
trybot_name = 'trybot-%s' % build |
|
if out and out.find(trybot_name) > 0: |
|
start_pos = out.find(trybot_name) |
|
end_pos = out.find('/chromiumos_test_image') |
|
trybot_image = out[start_pos:end_pos] |
|
|
|
return trybot_image |
|
|
|
|
|
def GetTrybotImage(chromeos_root, |
|
buildbot_name, |
|
patch_list, |
|
build_tag, |
|
other_flags=None, |
|
build_toolchain=False, |
|
async=False): |
|
"""Launch buildbot and get resulting trybot artifact name. |
|
|
|
This function launches a buildbot with the appropriate flags to |
|
build the test ChromeOS image, with the current ToT mobile compiler. It |
|
checks every 10 minutes to see if the trybot has finished. When the trybot |
|
has finished, it parses the resulting report logs to find the trybot |
|
artifact (if one was created), and returns that artifact name. |
|
|
|
chromeos_root is the path to the ChromeOS root, needed for finding chromite |
|
and launching the buildbot. |
|
|
|
buildbot_name is the name of the buildbot queue, such as lumpy-release or |
|
daisy-paladin. |
|
|
|
patch_list a python list of the patches, if any, for the buildbot to use. |
|
|
|
build_tag is a (unique) string to be used to look up the buildbot results |
|
from among all the build records. |
|
""" |
|
ce = command_executer.GetCommandExecuter() |
|
cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot') |
|
base_dir = os.getcwd() |
|
patch_arg = '' |
|
if patch_list: |
|
for p in patch_list: |
|
patch_arg = patch_arg + ' -g ' + repr(p) |
|
toolchain_flags = '' |
|
if build_toolchain: |
|
toolchain_flags += '--latest-toolchain' |
|
os.chdir(cbuildbot_path) |
|
if other_flags: |
|
optional_flags = ' '.join(other_flags) |
|
else: |
|
optional_flags = '' |
|
|
|
# Launch buildbot with appropriate flags. |
|
build = buildbot_name |
|
description = build_tag |
|
command_prefix = '' |
|
if not patch_arg: |
|
command_prefix = 'yes | ' |
|
command = ('%s ./cbuildbot --remote --nochromesdk %s' |
|
' --remote-description=%s %s %s %s' % (command_prefix, |
|
optional_flags, description, |
|
toolchain_flags, patch_arg, |
|
build)) |
|
_, out, _ = ce.RunCommandWOutput(command) |
|
if 'Tryjob submitted!' not in out: |
|
logger.GetLogger().LogFatal('Error occurred while launching trybot job: ' |
|
'%s' % command) |
|
|
|
os.chdir(base_dir) |
|
|
|
build_id = 0 |
|
build_status = None |
|
# Wait for buildbot to finish running (check every 10 minutes). Wait |
|
# 10 minutes before the first check to give the buildbot time to launch |
|
# (so we don't start looking for build data before it's out there). |
|
time.sleep(SLEEP_TIME) |
|
done = False |
|
pending = True |
|
# pending_time is the time between when we submit the job and when the |
|
# buildbot actually launches the build. running_time is the time between |
|
# when the buildbot job launches and when it finishes. The job is |
|
# considered 'pending' until we can find an entry for it in the buildbot |
|
# logs. |
|
pending_time = SLEEP_TIME |
|
running_time = 0 |
|
long_slept = False |
|
while not done: |
|
done = True |
|
build_info = GetBuildInfo(base_dir, build) |
|
if not build_info: |
|
if pending_time > TIME_OUT: |
|
logger.GetLogger().LogFatal('Unable to get build logs for target %s.' % |
|
build) |
|
else: |
|
pending_message = 'Unable to find build log; job may be pending.' |
|
done = False |
|
|
|
if done: |
|
data_dict = FindBuildRecordFromLog(description, build_info) |
|
if not data_dict: |
|
# Trybot job may be pending (not actually launched yet). |
|
if pending_time > TIME_OUT: |
|
logger.GetLogger().LogFatal('Unable to find build record for trybot' |
|
' %s.' % description) |
|
else: |
|
pending_message = 'Unable to find build record; job may be pending.' |
|
done = False |
|
|
|
else: |
|
# Now that we have actually found the entry for the build |
|
# job in the build log, we know the job is actually |
|
# runnning, not pending, so we flip the 'pending' flag. We |
|
# still have to wait for the buildbot job to finish running |
|
# however. |
|
pending = False |
|
build_id = data_dict['number'] |
|
|
|
if async: |
|
# Do not wait for trybot job to finish; return immediately |
|
return build_id, ' ' |
|
|
|
if not long_slept: |
|
# The trybot generally takes more than 2 hours to finish. |
|
# Wait two hours before polling the status. |
|
long_slept = True |
|
time.sleep(INITIAL_SLEEP_TIME) |
|
pending_time += INITIAL_SLEEP_TIME |
|
if True == data_dict['finished']: |
|
build_status = data_dict['results'] |
|
else: |
|
done = False |
|
|
|
if not done: |
|
if pending: |
|
logger.GetLogger().LogOutput(pending_message) |
|
logger.GetLogger().LogOutput('Current pending time: %d minutes.' % |
|
(pending_time / 60)) |
|
pending_time += SLEEP_TIME |
|
else: |
|
logger.GetLogger().LogOutput('{0} minutes passed.'.format(running_time / |
|
60)) |
|
logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME)) |
|
running_time += SLEEP_TIME |
|
|
|
time.sleep(SLEEP_TIME) |
|
if running_time > TIME_OUT: |
|
done = True |
|
|
|
trybot_image = '' |
|
|
|
if build.endswith('-toolchain'): |
|
# For rotating testers, we don't care about their build_status |
|
# result, because if any HWTest failed it will be non-zero. |
|
trybot_image = FindArchiveImage(chromeos_root, build, build_id) |
|
else: |
|
# The nightly performance tests do not run HWTests, so if |
|
# their build_status is non-zero, we do care. In this case |
|
# non-zero means the image itself probably did not build. |
|
if build_status in OK_STATUS: |
|
trybot_image = FindArchiveImage(chromeos_root, build, build_id) |
|
if not trybot_image: |
|
logger.GetLogger().LogError('Trybot job %s failed with status %d;' |
|
' no trybot image generated.' % |
|
(description, build_status)) |
|
|
|
logger.GetLogger().LogOutput("trybot_image is '%s'" % trybot_image) |
|
logger.GetLogger().LogOutput('build_status is %d' % build_status) |
|
return build_id, trybot_image |
|
|
|
|
|
def GetGSContent(chromeos_root, path): |
|
"""gsutil cat path""" |
|
|
|
ce = command_executer.GetCommandExecuter() |
|
command = ('gsutil cat gs://chromeos-image-archive/%s' % path) |
|
_, out, _ = ce.ChrootRunCommandWOutput( |
|
chromeos_root, command, print_to_console=False) |
|
return out |
|
|
|
|
|
def DoesImageExist(chromeos_root, build): |
|
"""Check if the image for the given build exists.""" |
|
|
|
ce = command_executer.GetCommandExecuter() |
|
command = ('gsutil ls gs://chromeos-image-archive/%s' |
|
'/chromiumos_test_image.tar.xz' % (build)) |
|
ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False) |
|
return not ret |
|
|
|
|
|
def WaitForImage(chromeos_root, build): |
|
"""Wait for an image to be ready.""" |
|
|
|
elapsed_time = 0 |
|
while elapsed_time < TIME_OUT: |
|
if DoesImageExist(chromeos_root, build): |
|
return |
|
logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' % |
|
build) |
|
time.sleep(SLEEP_TIME) |
|
elapsed_time += SLEEP_TIME |
|
|
|
logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' % |
|
(build, (TIME_OUT / 3600))) |
|
raise BuildbotTimeout('Timeout while waiting for image %s' % build)
|
|
|