You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
259 lines
7.2 KiB
259 lines
7.2 KiB
# Copyright 2011 Google Inc. All Rights Reserved. |
|
# |
|
|
|
__author__ = 'kbaclawski@google.com (Krystian Baclawski)' |
|
|
|
from collections import namedtuple |
|
import glob |
|
import gzip |
|
import os.path |
|
import pickle |
|
import time |
|
import xmlrpclib |
|
|
|
from django import forms |
|
from django.http import HttpResponseRedirect |
|
from django.shortcuts import render_to_response |
|
from django.template import Context |
|
from django.views import static |
|
|
|
Link = namedtuple('Link', 'href name') |
|
|
|
|
|
def GetServerConnection(): |
|
return xmlrpclib.Server('http://localhost:8000') |
|
|
|
|
|
def MakeDefaultContext(*args): |
|
context = Context({'links': [ |
|
Link('/job-group', 'Job Groups'), Link('/machine', 'Machines') |
|
]}) |
|
|
|
for arg in args: |
|
context.update(arg) |
|
|
|
return context |
|
|
|
|
|
class JobInfo(object): |
|
|
|
def __init__(self, job_id): |
|
self._job = pickle.loads(GetServerConnection().GetJob(job_id)) |
|
|
|
def GetAttributes(self): |
|
job = self._job |
|
|
|
group = [Link('/job-group/%d' % job.group.id, job.group.label)] |
|
|
|
predecessors = [Link('/job/%d' % pred.id, pred.label) |
|
for pred in job.predecessors] |
|
|
|
successors = [Link('/job/%d' % succ.id, succ.label) |
|
for succ in job.successors] |
|
|
|
machines = [Link('/machine/%s' % mach.hostname, mach.hostname) |
|
for mach in job.machines] |
|
|
|
logs = [Link('/job/%d/log' % job.id, 'Log')] |
|
|
|
commands = enumerate(job.PrettyFormatCommand().split('\n'), start=1) |
|
|
|
return {'text': [('Label', job.label), ('Directory', job.work_dir)], |
|
'link': [('Group', group), ('Predecessors', predecessors), |
|
('Successors', successors), ('Machines', machines), |
|
('Logs', logs)], |
|
'code': [('Command', commands)]} |
|
|
|
def GetTimeline(self): |
|
return [{'started': evlog.GetTimeStartedFormatted(), |
|
'state_from': evlog.event.from_, |
|
'state_to': evlog.event.to_, |
|
'elapsed': evlog.GetTimeElapsedRounded()} |
|
for evlog in self._job.timeline.GetTransitionEventHistory()] |
|
|
|
def GetLog(self): |
|
log_path = os.path.join(self._job.logs_dir, |
|
'%s.gz' % self._job.log_filename_prefix) |
|
|
|
try: |
|
log = gzip.open(log_path, 'r') |
|
except IOError: |
|
content = [] |
|
else: |
|
# There's a good chance that file is not closed yet, so EOF handling |
|
# function and CRC calculation will fail, thus we need to monkey patch the |
|
# _read_eof method. |
|
log._read_eof = lambda: None |
|
|
|
def SplitLine(line): |
|
prefix, msg = line.split(': ', 1) |
|
datetime, stream = prefix.rsplit(' ', 1) |
|
|
|
return datetime, stream, msg |
|
|
|
content = map(SplitLine, log.readlines()) |
|
finally: |
|
log.close() |
|
|
|
return content |
|
|
|
|
|
class JobGroupInfo(object): |
|
|
|
def __init__(self, job_group_id): |
|
self._job_group = pickle.loads(GetServerConnection().GetJobGroup( |
|
job_group_id)) |
|
|
|
def GetAttributes(self): |
|
group = self._job_group |
|
|
|
home_dir = [Link('/job-group/%d/files/' % group.id, group.home_dir)] |
|
|
|
return {'text': [('Label', group.label), |
|
('Time submitted', time.ctime(group.time_submitted)), |
|
('State', group.status), |
|
('Cleanup on completion', group.cleanup_on_completion), |
|
('Cleanup on failure', group.cleanup_on_failure)], |
|
'link': [('Directory', home_dir)]} |
|
|
|
def _GetJobStatus(self, job): |
|
status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'} |
|
return status_map.get(str(job.status), None) |
|
|
|
def GetJobList(self): |
|
return [{'id': job.id, |
|
'label': job.label, |
|
'state': job.status, |
|
'status': self._GetJobStatus(job), |
|
'elapsed': job.timeline.GetTotalTime()} |
|
for job in self._job_group.jobs] |
|
|
|
def GetHomeDirectory(self): |
|
return self._job_group.home_dir |
|
|
|
def GetReportList(self): |
|
job_dir_pattern = os.path.join(self._job_group.home_dir, 'job-*') |
|
|
|
filenames = [] |
|
|
|
for job_dir in glob.glob(job_dir_pattern): |
|
filename = os.path.join(job_dir, 'report.html') |
|
|
|
if os.access(filename, os.F_OK): |
|
filenames.append(filename) |
|
|
|
reports = [] |
|
|
|
for filename in sorted(filenames, key=lambda f: os.stat(f).st_ctime): |
|
try: |
|
with open(filename, 'r') as report: |
|
reports.append(report.read()) |
|
except IOError: |
|
pass |
|
|
|
return reports |
|
|
|
|
|
class JobGroupListInfo(object): |
|
|
|
def __init__(self): |
|
self._all_job_groups = pickle.loads(GetServerConnection().GetAllJobGroups()) |
|
|
|
def _GetJobGroupState(self, group): |
|
return str(group.status) |
|
|
|
def _GetJobGroupStatus(self, group): |
|
status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'} |
|
return status_map.get(self._GetJobGroupState(group), None) |
|
|
|
def GetList(self): |
|
return [{'id': group.id, |
|
'label': group.label, |
|
'submitted': time.ctime(group.time_submitted), |
|
'state': self._GetJobGroupState(group), |
|
'status': self._GetJobGroupStatus(group)} |
|
for group in self._all_job_groups] |
|
|
|
def GetLabelList(self): |
|
return sorted(set(group.label for group in self._all_job_groups)) |
|
|
|
|
|
def JobPageHandler(request, job_id): |
|
job = JobInfo(int(job_id)) |
|
|
|
ctx = MakeDefaultContext({ |
|
'job_id': job_id, |
|
'attributes': job.GetAttributes(), |
|
'timeline': job.GetTimeline() |
|
}) |
|
|
|
return render_to_response('job.html', ctx) |
|
|
|
|
|
def LogPageHandler(request, job_id): |
|
job = JobInfo(int(job_id)) |
|
|
|
ctx = MakeDefaultContext({'job_id': job_id, 'log_lines': job.GetLog()}) |
|
|
|
return render_to_response('job_log.html', ctx) |
|
|
|
|
|
def JobGroupPageHandler(request, job_group_id): |
|
group = JobGroupInfo(int(job_group_id)) |
|
|
|
ctx = MakeDefaultContext({ |
|
'group_id': job_group_id, |
|
'attributes': group.GetAttributes(), |
|
'job_list': group.GetJobList(), |
|
'reports': group.GetReportList() |
|
}) |
|
|
|
return render_to_response('job_group.html', ctx) |
|
|
|
|
|
def JobGroupFilesPageHandler(request, job_group_id, path): |
|
group = JobGroupInfo(int(job_group_id)) |
|
|
|
return static.serve(request, |
|
path, |
|
document_root=group.GetHomeDirectory(), |
|
show_indexes=True) |
|
|
|
|
|
class FilterJobGroupsForm(forms.Form): |
|
label = forms.ChoiceField(label='Filter by label:', required=False) |
|
|
|
|
|
def JobGroupListPageHandler(request): |
|
groups = JobGroupListInfo() |
|
group_list = groups.GetList() |
|
|
|
field = FilterJobGroupsForm.base_fields['label'] |
|
field.choices = [('*', '--- no filtering ---')] |
|
field.choices.extend([(label, label) for label in groups.GetLabelList()]) |
|
|
|
if request.method == 'POST': |
|
form = FilterJobGroupsForm(request.POST) |
|
|
|
if form.is_valid(): |
|
label = form.cleaned_data['label'] |
|
|
|
if label != '*': |
|
group_list = [group for group in group_list if group['label'] == label] |
|
else: |
|
form = FilterJobGroupsForm({'initial': '*'}) |
|
|
|
ctx = MakeDefaultContext({'filter': form, 'groups': group_list}) |
|
|
|
return render_to_response('job_group_list.html', ctx) |
|
|
|
|
|
def MachineListPageHandler(request): |
|
machine_list = pickle.loads(GetServerConnection().GetMachineList()) |
|
|
|
return render_to_response('machine_list.html', |
|
MakeDefaultContext({'machines': machine_list})) |
|
|
|
|
|
def DefaultPageHandler(request): |
|
return HttpResponseRedirect('/job-group')
|
|
|