You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
269 lines
9.6 KiB
269 lines
9.6 KiB
#!/usr/bin/env python |
|
# Copyright 2016 The PDFium Authors. All rights reserved. |
|
# Use of this source code is governed by a BSD-style license that can be |
|
# found in the LICENSE file. |
|
|
|
import cStringIO |
|
import functools |
|
import multiprocessing |
|
import optparse |
|
import os |
|
import re |
|
import shutil |
|
import subprocess |
|
import sys |
|
|
|
import common |
|
import gold |
|
import pngdiffer |
|
import suppressor |
|
|
|
class KeyboardInterruptError(Exception): pass |
|
|
|
# Nomenclature: |
|
# x_root - "x" |
|
# x_filename - "x.ext" |
|
# x_path - "path/to/a/b/c/x.ext" |
|
# c_dir - "path/to/a/b/c" |
|
|
|
def TestOneFileParallel(this, test_case): |
|
"""Wrapper to call GenerateAndTest() and redirect output to stdout.""" |
|
try: |
|
input_filename, source_dir = test_case |
|
result = this.GenerateAndTest(input_filename, source_dir); |
|
return (result, input_filename, source_dir) |
|
except KeyboardInterrupt: |
|
raise KeyboardInterruptError() |
|
|
|
|
|
class TestRunner: |
|
def __init__(self, dirname): |
|
self.test_dir = dirname |
|
|
|
# GenerateAndTest returns a tuple <success, outputfiles> where |
|
# success is a boolean indicating whether the tests passed comparison |
|
# tests and outputfiles is a list tuples: |
|
# (path_to_image, md5_hash_of_pixelbuffer) |
|
def GenerateAndTest(self, input_filename, source_dir): |
|
input_root, _ = os.path.splitext(input_filename) |
|
expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt') |
|
|
|
pdf_path = os.path.join(self.working_dir, input_root + '.pdf') |
|
|
|
# Remove any existing generated images from previous runs. |
|
actual_images = self.image_differ.GetActualFiles(input_filename, source_dir, |
|
self.working_dir) |
|
for image in actual_images: |
|
if os.path.exists(image): |
|
os.remove(image) |
|
|
|
sys.stdout.flush() |
|
|
|
raised_exception = self.Generate(source_dir, input_filename, input_root, |
|
pdf_path) |
|
|
|
if raised_exception != None: |
|
print "FAILURE: " + input_filename + "; " + str(raised_exception) |
|
return False, [] |
|
|
|
results = [] |
|
if os.path.exists(expected_txt_path): |
|
raised_exception = self.TestText(input_root, expected_txt_path, pdf_path) |
|
else: |
|
raised_exception, results = self.TestPixel(input_root, pdf_path) |
|
|
|
if raised_exception != None: |
|
print "FAILURE: " + input_filename + "; " + str(raised_exception) |
|
return False, results |
|
|
|
if len(actual_images): |
|
if self.image_differ.HasDifferences(input_filename, source_dir, |
|
self.working_dir): |
|
return False, results |
|
return True, results |
|
|
|
def Generate(self, source_dir, input_filename, input_root, pdf_path): |
|
original_path = os.path.join(source_dir, input_filename) |
|
input_path = os.path.join(source_dir, input_root + '.in') |
|
|
|
input_event_path = os.path.join(source_dir, input_root + ".evt") |
|
if os.path.exists(input_event_path): |
|
output_event_path = os.path.splitext(pdf_path)[0] + ".evt" |
|
shutil.copyfile(input_event_path, output_event_path) |
|
|
|
if not os.path.exists(input_path): |
|
if os.path.exists(original_path): |
|
shutil.copyfile(original_path, pdf_path) |
|
return None |
|
|
|
sys.stdout.flush() |
|
|
|
return common.RunCommand( |
|
[sys.executable, self.fixup_path, '--output-dir=' + self.working_dir, |
|
input_path]) |
|
|
|
|
|
def TestText(self, input_root, expected_txt_path, pdf_path): |
|
txt_path = os.path.join(self.working_dir, input_root + '.txt') |
|
|
|
with open(txt_path, 'w') as outfile: |
|
cmd_to_run = [self.pdfium_test_path, pdf_path] |
|
subprocess.check_call(cmd_to_run, stdout=outfile) |
|
|
|
cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path] |
|
return common.RunCommand(cmd) |
|
|
|
|
|
def TestPixel(self, input_root, pdf_path): |
|
cmd_to_run = [self.pdfium_test_path, '--send-events', '--png'] |
|
if self.gold_results: |
|
cmd_to_run.append('--md5') |
|
cmd_to_run.append(pdf_path) |
|
return common.RunCommandExtractHashedFiles(cmd_to_run) |
|
|
|
def HandleResult(self, input_filename, input_path, result): |
|
success, image_paths = result |
|
if self.gold_results: |
|
if image_paths: |
|
for img_path, md5_hash in image_paths: |
|
# the output filename (without extension becomes the test name) |
|
test_name = os.path.splitext(os.path.split(img_path)[1])[0] |
|
self.gold_results.AddTestResult(test_name, md5_hash, img_path) |
|
|
|
if self.test_suppressor.IsResultSuppressed(input_filename): |
|
if success: |
|
self.surprises.append(input_path) |
|
else: |
|
if not success: |
|
self.failures.append(input_path) |
|
|
|
|
|
def Run(self): |
|
parser = optparse.OptionParser() |
|
|
|
parser.add_option('--build-dir', default=os.path.join('out', 'Debug'), |
|
help='relative path from the base source directory') |
|
|
|
parser.add_option('-j', default=multiprocessing.cpu_count(), |
|
dest='num_workers', type='int', |
|
help='run NUM_WORKERS jobs in parallel') |
|
|
|
parser.add_option('--gold_properties', default='', dest="gold_properties", |
|
help='Key value pairs that are written to the top level of the JSON file that is ingested by Gold.') |
|
|
|
parser.add_option('--gold_key', default='', dest="gold_key", |
|
help='Key value pairs that are added to the "key" field of the JSON file that is ingested by Gold.') |
|
|
|
parser.add_option('--gold_output_dir', default='', dest="gold_output_dir", |
|
help='Path of where to write the JSON output to be uploaded to Gold.') |
|
|
|
parser.add_option('--gold_ignore_hashes', default='', dest="gold_ignore_hashes", |
|
help='Path to a file with MD5 hashes we wish to ignore.') |
|
|
|
parser.add_option('--ignore_errors', action="store_true", dest="ignore_errors", |
|
help='Prevents the return value from being non-zero when image comparison fails.') |
|
|
|
options, args = parser.parse_args() |
|
|
|
finder = common.DirectoryFinder(options.build_dir) |
|
self.fixup_path = finder.ScriptPath('fixup_pdf_template.py') |
|
self.text_diff_path = finder.ScriptPath('text_diff.py') |
|
|
|
self.source_dir = finder.TestingDir() |
|
if self.test_dir != 'corpus': |
|
test_dir = finder.TestingDir(os.path.join('resources', self.test_dir)) |
|
else: |
|
test_dir = finder.TestingDir(self.test_dir) |
|
|
|
self.pdfium_test_path = finder.ExecutablePath('pdfium_test') |
|
if not os.path.exists(self.pdfium_test_path): |
|
print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path |
|
print "Use --build-dir to specify its location." |
|
return 1 |
|
|
|
self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir)) |
|
if not os.path.exists(self.working_dir): |
|
os.makedirs(self.working_dir) |
|
|
|
self.feature_string = subprocess.check_output([self.pdfium_test_path, |
|
'--show-config']) |
|
self.test_suppressor = suppressor.Suppressor(finder, self.feature_string) |
|
self.image_differ = pngdiffer.PNGDiffer(finder) |
|
|
|
walk_from_dir = finder.TestingDir(test_dir); |
|
|
|
test_cases = [] |
|
input_file_re = re.compile('^[a-zA-Z0-9_.]+[.](in|pdf)$') |
|
if len(args): |
|
for file_name in args: |
|
file_name.replace(".pdf", ".in") |
|
input_path = os.path.join(walk_from_dir, file_name) |
|
if not os.path.isfile(input_path): |
|
print "Can't find test file '%s'" % file_name |
|
return 1 |
|
|
|
test_cases.append((os.path.basename(input_path), |
|
os.path.dirname(input_path))) |
|
else: |
|
for file_dir, _, filename_list in os.walk(walk_from_dir): |
|
for input_filename in filename_list: |
|
if input_file_re.match(input_filename): |
|
input_path = os.path.join(file_dir, input_filename) |
|
if not self.test_suppressor.IsExecutionSuppressed(input_path): |
|
if os.path.isfile(input_path): |
|
test_cases.append((input_filename, file_dir)) |
|
|
|
self.failures = [] |
|
self.surprises = [] |
|
|
|
# Collect Gold results if an output directory was named. |
|
self.gold_results = None |
|
if options.gold_output_dir: |
|
self.gold_results = gold.GoldResults("pdfium", |
|
options.gold_output_dir, |
|
options.gold_properties, |
|
options.gold_key, |
|
options.gold_ignore_hashes) |
|
|
|
if options.num_workers > 1 and len(test_cases) > 1: |
|
try: |
|
pool = multiprocessing.Pool(options.num_workers) |
|
worker_func = functools.partial(TestOneFileParallel, self) |
|
|
|
worker_results = pool.imap(worker_func, test_cases) |
|
for worker_result in worker_results: |
|
result, input_filename, source_dir = worker_result |
|
input_path = os.path.join(source_dir, input_filename) |
|
|
|
self.HandleResult(input_filename, input_path, result) |
|
|
|
except KeyboardInterrupt: |
|
pool.terminate() |
|
finally: |
|
pool.close() |
|
pool.join() |
|
else: |
|
for test_case in test_cases: |
|
input_filename, input_file_dir = test_case |
|
result = self.GenerateAndTest(input_filename, input_file_dir) |
|
self.HandleResult(input_filename, |
|
os.path.join(input_file_dir, input_filename), result) |
|
|
|
if self.gold_results: |
|
self.gold_results.WriteResults() |
|
|
|
if self.surprises: |
|
self.surprises.sort() |
|
print '\n\nUnexpected Successes:' |
|
for surprise in self.surprises: |
|
print surprise; |
|
|
|
if self.failures: |
|
self.failures.sort() |
|
print '\n\nSummary of Failures:' |
|
for failure in self.failures: |
|
print failure |
|
|
|
if not options.ignore_errors: |
|
return 1 |
|
return 0
|
|
|