You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
447 lines
14 KiB
447 lines
14 KiB
#!/usr/bin/python2 |
|
"""Module of binary serch for perforce.""" |
|
from __future__ import print_function |
|
|
|
import math |
|
import argparse |
|
import os |
|
import re |
|
import sys |
|
import tempfile |
|
|
|
from cros_utils import command_executer |
|
from cros_utils import logger |
|
|
|
verbose = True |
|
|
|
|
|
def _GetP4ClientSpec(client_name, p4_paths): |
|
p4_string = '' |
|
for p4_path in p4_paths: |
|
if ' ' not in p4_path: |
|
p4_string += ' -a %s' % p4_path |
|
else: |
|
p4_string += " -a \"" + (' //' + client_name + '/').join(p4_path) + "\"" |
|
|
|
return p4_string |
|
|
|
|
|
def GetP4Command(client_name, p4_port, p4_paths, checkoutdir, p4_snapshot=''): |
|
command = '' |
|
|
|
if p4_snapshot: |
|
command += 'mkdir -p ' + checkoutdir |
|
for p4_path in p4_paths: |
|
real_path = p4_path[1] |
|
if real_path.endswith('...'): |
|
real_path = real_path.replace('/...', '') |
|
command += ( |
|
'; mkdir -p ' + checkoutdir + '/' + os.path.dirname(real_path)) |
|
command += ('&& rsync -lr ' + p4_snapshot + '/' + real_path + ' ' + |
|
checkoutdir + '/' + os.path.dirname(real_path)) |
|
return command |
|
|
|
command += ' export P4CONFIG=.p4config' |
|
command += ' && mkdir -p ' + checkoutdir |
|
command += ' && cd ' + checkoutdir |
|
command += ' && cp ${HOME}/.p4config .' |
|
command += ' && chmod u+w .p4config' |
|
command += " && echo \"P4PORT=" + p4_port + "\" >> .p4config" |
|
command += " && echo \"P4CLIENT=" + client_name + "\" >> .p4config" |
|
command += (' && g4 client ' + _GetP4ClientSpec(client_name, p4_paths)) |
|
command += ' && g4 sync ' |
|
command += ' && cd -' |
|
return command |
|
|
|
|
|
class BinarySearchPoint(object): |
|
"""Class of binary search point.""" |
|
|
|
def __init__(self, revision, status, tag=None): |
|
self.revision = revision |
|
self.status = status |
|
self.tag = tag |
|
|
|
|
|
class BinarySearcher(object): |
|
"""Class of binary searcher.""" |
|
|
|
def __init__(self, logger_to_set=None): |
|
self.sorted_list = [] |
|
self.index_log = [] |
|
self.status_log = [] |
|
self.skipped_indices = [] |
|
self.current = 0 |
|
self.points = {} |
|
self.lo = 0 |
|
self.hi = 0 |
|
if logger_to_set is not None: |
|
self.logger = logger_to_set |
|
else: |
|
self.logger = logger.GetLogger() |
|
|
|
def SetSortedList(self, sorted_list): |
|
assert len(sorted_list) > 0 |
|
self.sorted_list = sorted_list |
|
self.index_log = [] |
|
self.hi = len(sorted_list) - 1 |
|
self.lo = 0 |
|
self.points = {} |
|
for i in range(len(self.sorted_list)): |
|
bsp = BinarySearchPoint(self.sorted_list[i], -1, 'Not yet done.') |
|
self.points[i] = bsp |
|
|
|
def SetStatus(self, status, tag=None): |
|
message = ('Revision: %s index: %d returned: %d' % |
|
(self.sorted_list[self.current], self.current, status)) |
|
self.logger.LogOutput(message, print_to_console=verbose) |
|
assert status == 0 or status == 1 or status == 125 |
|
self.index_log.append(self.current) |
|
self.status_log.append(status) |
|
bsp = BinarySearchPoint(self.sorted_list[self.current], status, tag) |
|
self.points[self.current] = bsp |
|
|
|
if status == 125: |
|
self.skipped_indices.append(self.current) |
|
|
|
if status == 0 or status == 1: |
|
if status == 0: |
|
self.lo = self.current + 1 |
|
elif status == 1: |
|
self.hi = self.current |
|
self.logger.LogOutput('lo: %d hi: %d\n' % (self.lo, self.hi)) |
|
self.current = (self.lo + self.hi) / 2 |
|
|
|
if self.lo == self.hi: |
|
message = ('Search complete. First bad version: %s' |
|
' at index: %d' % (self.sorted_list[self.current], self.lo)) |
|
self.logger.LogOutput(message) |
|
return True |
|
|
|
for index in range(self.lo, self.hi): |
|
if index not in self.skipped_indices: |
|
return False |
|
self.logger.LogOutput( |
|
'All skipped indices between: %d and %d\n' % (self.lo, self.hi), |
|
print_to_console=verbose) |
|
return True |
|
|
|
# Does a better job with chromeos flakiness. |
|
def GetNextFlakyBinary(self): |
|
t = (self.lo, self.current, self.hi) |
|
q = [t] |
|
while len(q): |
|
element = q.pop(0) |
|
if element[1] in self.skipped_indices: |
|
# Go top |
|
to_add = (element[0], (element[0] + element[1]) / 2, element[1]) |
|
q.append(to_add) |
|
# Go bottom |
|
to_add = (element[1], (element[1] + element[2]) / 2, element[2]) |
|
q.append(to_add) |
|
else: |
|
self.current = element[1] |
|
return |
|
assert len(q), 'Queue should never be 0-size!' |
|
|
|
def GetNextFlakyLinear(self): |
|
current_hi = self.current |
|
current_lo = self.current |
|
while True: |
|
if current_hi < self.hi and current_hi not in self.skipped_indices: |
|
self.current = current_hi |
|
break |
|
if current_lo >= self.lo and current_lo not in self.skipped_indices: |
|
self.current = current_lo |
|
break |
|
if current_lo < self.lo and current_hi >= self.hi: |
|
break |
|
|
|
current_hi += 1 |
|
current_lo -= 1 |
|
|
|
def GetNext(self): |
|
self.current = (self.hi + self.lo) / 2 |
|
# Try going forward if current is skipped. |
|
if self.current in self.skipped_indices: |
|
self.GetNextFlakyBinary() |
|
|
|
# TODO: Add an estimated time remaining as well. |
|
message = ('Estimated tries: min: %d max: %d\n' % |
|
(1 + math.log(self.hi - self.lo, 2), |
|
self.hi - self.lo - len(self.skipped_indices))) |
|
self.logger.LogOutput(message, print_to_console=verbose) |
|
message = ('lo: %d hi: %d current: %d version: %s\n' % |
|
(self.lo, self.hi, self.current, self.sorted_list[self.current])) |
|
self.logger.LogOutput(message, print_to_console=verbose) |
|
self.logger.LogOutput(str(self), print_to_console=verbose) |
|
return self.sorted_list[self.current] |
|
|
|
def SetLoRevision(self, lo_revision): |
|
self.lo = self.sorted_list.index(lo_revision) |
|
|
|
def SetHiRevision(self, hi_revision): |
|
self.hi = self.sorted_list.index(hi_revision) |
|
|
|
def GetAllPoints(self): |
|
to_return = '' |
|
for i in range(len(self.sorted_list)): |
|
to_return += ('%d %d %s\n' % (self.points[i].status, i, |
|
self.points[i].revision)) |
|
|
|
return to_return |
|
|
|
def __str__(self): |
|
to_return = '' |
|
to_return += 'Current: %d\n' % self.current |
|
to_return += str(self.index_log) + '\n' |
|
revision_log = [] |
|
for index in self.index_log: |
|
revision_log.append(self.sorted_list[index]) |
|
to_return += str(revision_log) + '\n' |
|
to_return += str(self.status_log) + '\n' |
|
to_return += 'Skipped indices:\n' |
|
to_return += str(self.skipped_indices) + '\n' |
|
to_return += self.GetAllPoints() |
|
return to_return |
|
|
|
|
|
class RevisionInfo(object): |
|
"""Class of reversion info.""" |
|
|
|
def __init__(self, date, client, description): |
|
self.date = date |
|
self.client = client |
|
self.description = description |
|
self.status = -1 |
|
|
|
|
|
class VCSBinarySearcher(object): |
|
"""Class of VCS binary searcher.""" |
|
|
|
def __init__(self): |
|
self.bs = BinarySearcher() |
|
self.rim = {} |
|
self.current_ce = None |
|
self.checkout_dir = None |
|
self.current_revision = None |
|
|
|
def Initialize(self): |
|
pass |
|
|
|
def GetNextRevision(self): |
|
pass |
|
|
|
def CheckoutRevision(self, revision): |
|
pass |
|
|
|
def SetStatus(self, status): |
|
pass |
|
|
|
def Cleanup(self): |
|
pass |
|
|
|
def SetGoodRevision(self, revision): |
|
if revision is None: |
|
return |
|
assert revision in self.bs.sorted_list |
|
self.bs.SetLoRevision(revision) |
|
|
|
def SetBadRevision(self, revision): |
|
if revision is None: |
|
return |
|
assert revision in self.bs.sorted_list |
|
self.bs.SetHiRevision(revision) |
|
|
|
|
|
class P4BinarySearcher(VCSBinarySearcher): |
|
"""Class of P4 binary searcher.""" |
|
|
|
def __init__(self, p4_port, p4_paths, test_command): |
|
VCSBinarySearcher.__init__(self) |
|
self.p4_port = p4_port |
|
self.p4_paths = p4_paths |
|
self.test_command = test_command |
|
self.checkout_dir = tempfile.mkdtemp() |
|
self.ce = command_executer.GetCommandExecuter() |
|
self.client_name = 'binary-searcher-$HOSTNAME-$USER' |
|
self.job_log_root = '/home/asharif/www/coreboot_triage/' |
|
self.changes = None |
|
|
|
def Initialize(self): |
|
self.Cleanup() |
|
command = GetP4Command(self.client_name, self.p4_port, self.p4_paths, 1, |
|
self.checkout_dir) |
|
self.ce.RunCommand(command) |
|
command = 'cd %s && g4 changes ...' % self.checkout_dir |
|
_, out, _ = self.ce.RunCommandWOutput(command) |
|
self.changes = re.findall(r'Change (\d+)', out) |
|
change_infos = re.findall(r'Change (\d+) on ([\d/]+) by ' |
|
r"([^\s]+) ('[^']*')", out) |
|
for change_info in change_infos: |
|
ri = RevisionInfo(change_info[1], change_info[2], change_info[3]) |
|
self.rim[change_info[0]] = ri |
|
# g4 gives changes in reverse chronological order. |
|
self.changes.reverse() |
|
self.bs.SetSortedList(self.changes) |
|
|
|
def SetStatus(self, status): |
|
self.rim[self.current_revision].status = status |
|
return self.bs.SetStatus(status) |
|
|
|
def GetNextRevision(self): |
|
next_revision = self.bs.GetNext() |
|
self.current_revision = next_revision |
|
return next_revision |
|
|
|
def CleanupCLs(self): |
|
if not os.path.isfile(self.checkout_dir + '/.p4config'): |
|
command = 'cd %s' % self.checkout_dir |
|
command += ' && cp ${HOME}/.p4config .' |
|
command += " && echo \"P4PORT=" + self.p4_port + "\" >> .p4config" |
|
command += " && echo \"P4CLIENT=" + self.client_name + "\" >> .p4config" |
|
self.ce.RunCommand(command) |
|
command = 'cd %s' % self.checkout_dir |
|
command += '; g4 changes -c %s' % self.client_name |
|
_, out, _ = self.ce.RunCommandWOUTPUOT(command) |
|
changes = re.findall(r'Change (\d+)', out) |
|
if len(changes) != 0: |
|
command = 'cd %s' % self.checkout_dir |
|
for change in changes: |
|
command += '; g4 revert -c %s' % change |
|
self.ce.RunCommand(command) |
|
|
|
def CleanupClient(self): |
|
command = 'cd %s' % self.checkout_dir |
|
command += '; g4 revert ...' |
|
command += '; g4 client -d %s' % self.client_name |
|
self.ce.RunCommand(command) |
|
|
|
def Cleanup(self): |
|
self.CleanupCLs() |
|
self.CleanupClient() |
|
|
|
def __str__(self): |
|
to_return = '' |
|
for change in self.changes: |
|
ri = self.rim[change] |
|
if ri.status == -1: |
|
to_return = '%s\t%d\n' % (change, ri.status) |
|
else: |
|
to_return += ('%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\n' % |
|
(change, ri.status, ri.date, ri.client, ri.description, |
|
self.job_log_root + change + '.cmd', |
|
self.job_log_root + change + '.out', |
|
self.job_log_root + change + '.err')) |
|
return to_return |
|
|
|
|
|
class P4GCCBinarySearcher(P4BinarySearcher): |
|
"""Class of P4 gcc binary searcher.""" |
|
|
|
# TODO: eventually get these patches from g4 instead of creating them manually |
|
def HandleBrokenCLs(self, current_revision): |
|
cr = int(current_revision) |
|
problematic_ranges = [] |
|
problematic_ranges.append([44528, 44539]) |
|
problematic_ranges.append([44528, 44760]) |
|
problematic_ranges.append([44335, 44882]) |
|
command = 'pwd' |
|
for pr in problematic_ranges: |
|
if cr in range(pr[0], pr[1]): |
|
patch_file = '/home/asharif/triage_tool/%d-%d.patch' % (pr[0], pr[1]) |
|
f = open(patch_file) |
|
patch = f.read() |
|
f.close() |
|
files = re.findall('--- (//.*)', patch) |
|
command += '; cd %s' % self.checkout_dir |
|
for f in files: |
|
command += '; g4 open %s' % f |
|
command += '; patch -p2 < %s' % patch_file |
|
self.current_ce.RunCommand(command) |
|
|
|
def CheckoutRevision(self, current_revision): |
|
job_logger = logger.Logger( |
|
self.job_log_root, current_revision, True, subdir='') |
|
self.current_ce = command_executer.GetCommandExecuter(job_logger) |
|
|
|
self.CleanupCLs() |
|
# Change the revision of only the gcc part of the toolchain. |
|
command = ('cd %s/gcctools/google_vendor_src_branch/gcc ' |
|
'&& g4 revert ...; g4 sync @%s' % |
|
(self.checkout_dir, current_revision)) |
|
self.current_ce.RunCommand(command) |
|
|
|
self.HandleBrokenCLs(current_revision) |
|
|
|
|
|
def Main(argv): |
|
"""The main function.""" |
|
# Common initializations |
|
### command_executer.InitCommandExecuter(True) |
|
ce = command_executer.GetCommandExecuter() |
|
|
|
parser = argparse.ArgumentParser() |
|
parser.add_argument( |
|
'-n', |
|
'--num_tries', |
|
dest='num_tries', |
|
default='100', |
|
help='Number of tries.') |
|
parser.add_argument( |
|
'-g', |
|
'--good_revision', |
|
dest='good_revision', |
|
help='Last known good revision.') |
|
parser.add_argument( |
|
'-b', |
|
'--bad_revision', |
|
dest='bad_revision', |
|
help='Last known bad revision.') |
|
parser.add_argument( |
|
'-s', '--script', dest='script', help='Script to run for every version.') |
|
options = parser.parse_args(argv) |
|
# First get all revisions |
|
p4_paths = ['//depot2/gcctools/google_vendor_src_branch/gcc/gcc-4.4.3/...', |
|
'//depot2/gcctools/google_vendor_src_branch/binutils/' |
|
'binutils-2.20.1-mobile/...', |
|
'//depot2/gcctools/google_vendor_src_branch/' |
|
'binutils/binutils-20100303/...'] |
|
p4gccbs = P4GCCBinarySearcher('perforce2:2666', p4_paths, '') |
|
|
|
# Main loop: |
|
terminated = False |
|
num_tries = int(options.num_tries) |
|
script = os.path.expanduser(options.script) |
|
|
|
try: |
|
p4gccbs.Initialize() |
|
p4gccbs.SetGoodRevision(options.good_revision) |
|
p4gccbs.SetBadRevision(options.bad_revision) |
|
while not terminated and num_tries > 0: |
|
current_revision = p4gccbs.GetNextRevision() |
|
|
|
# Now run command to get the status |
|
ce = command_executer.GetCommandExecuter() |
|
command = '%s %s' % (script, p4gccbs.checkout_dir) |
|
status = ce.RunCommand(command) |
|
message = ('Revision: %s produced: %d status\n' % |
|
(current_revision, status)) |
|
logger.GetLogger().LogOutput(message, print_to_console=verbose) |
|
terminated = p4gccbs.SetStatus(status) |
|
num_tries -= 1 |
|
logger.GetLogger().LogOutput(str(p4gccbs), print_to_console=verbose) |
|
|
|
if not terminated: |
|
logger.GetLogger().LogOutput( |
|
'Tries: %d expired.' % num_tries, print_to_console=verbose) |
|
logger.GetLogger().LogOutput(str(p4gccbs.bs), print_to_console=verbose) |
|
except (KeyboardInterrupt, SystemExit): |
|
logger.GetLogger().LogOutput('Cleaning up...') |
|
finally: |
|
logger.GetLogger().LogOutput(str(p4gccbs.bs), print_to_console=verbose) |
|
status = p4gccbs.Cleanup() |
|
|
|
|
|
if __name__ == '__main__': |
|
Main(sys.argv[1:])
|
|
|