You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
534 lines
18 KiB
534 lines
18 KiB
# -*- coding: utf-8 -*- |
|
|
|
#------------------------------------------------------------------------- |
|
# drawElements Quality Program utilities |
|
# -------------------------------------- |
|
# |
|
# Copyright 2015 The Android Open Source Project |
|
# |
|
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
# you may not use this file except in compliance with the License. |
|
# You may obtain a copy of the License at |
|
# |
|
# http://www.apache.org/licenses/LICENSE-2.0 |
|
# |
|
# Unless required by applicable law or agreed to in writing, software |
|
# distributed under the License is distributed on an "AS IS" BASIS, |
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
# See the License for the specific language governing permissions and |
|
# limitations under the License. |
|
# |
|
#------------------------------------------------------------------------- |
|
|
|
import os |
|
import re |
|
import sys |
|
import copy |
|
import zlib |
|
import time |
|
import shlex |
|
import shutil |
|
import fnmatch |
|
import tarfile |
|
import argparse |
|
import platform |
|
import datetime |
|
import tempfile |
|
import posixpath |
|
import subprocess |
|
|
|
from build.common import * |
|
from build.config import * |
|
from build.build import * |
|
|
|
def die (msg): |
|
print msg |
|
sys.exit(-1) |
|
|
|
def removeLeadingPath (path, basePath): |
|
# Both inputs must be normalized already |
|
assert os.path.normpath(path) == path |
|
assert os.path.normpath(basePath) == basePath |
|
return path[len(basePath) + 1:] |
|
|
|
def findFile (candidates): |
|
for file in candidates: |
|
if os.path.exists(file): |
|
return file |
|
return None |
|
|
|
def getFileList (basePath): |
|
allFiles = [] |
|
basePath = os.path.normpath(basePath) |
|
for root, dirs, files in os.walk(basePath): |
|
for file in files: |
|
relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath) |
|
allFiles.append(relPath) |
|
return allFiles |
|
|
|
def toDatetime (dateTuple): |
|
Y, M, D = dateTuple |
|
return datetime.datetime(Y, M, D) |
|
|
|
class PackageBuildInfo: |
|
def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath): |
|
self.releaseConfig = releaseConfig |
|
self.srcBasePath = srcBasePath |
|
self.dstBasePath = dstBasePath |
|
self.tmpBasePath = tmpBasePath |
|
|
|
def getReleaseConfig (self): |
|
return self.releaseConfig |
|
|
|
def getReleaseVersion (self): |
|
return self.releaseConfig.getVersion() |
|
|
|
def getReleaseId (self): |
|
# Release id is crc32(releaseConfig + release) |
|
return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff |
|
|
|
def getSrcBasePath (self): |
|
return self.srcBasePath |
|
|
|
def getTmpBasePath (self): |
|
return self.tmpBasePath |
|
|
|
class DstFile (object): |
|
def __init__ (self, dstFile): |
|
self.dstFile = dstFile |
|
|
|
def makeDir (self): |
|
dirName = os.path.dirname(self.dstFile) |
|
if not os.path.exists(dirName): |
|
os.makedirs(dirName) |
|
|
|
def make (self, packageBuildInfo): |
|
assert False # Should not be called |
|
|
|
class CopyFile (DstFile): |
|
def __init__ (self, srcFile, dstFile): |
|
super(CopyFile, self).__init__(dstFile) |
|
self.srcFile = srcFile |
|
|
|
def make (self, packageBuildInfo): |
|
self.makeDir() |
|
if os.path.exists(self.dstFile): |
|
die("%s already exists" % self.dstFile) |
|
shutil.copyfile(self.srcFile, self.dstFile) |
|
|
|
class GenReleaseInfoFileTarget (DstFile): |
|
def __init__ (self, dstFile): |
|
super(GenReleaseInfoFileTarget, self).__init__(dstFile) |
|
|
|
def make (self, packageBuildInfo): |
|
self.makeDir() |
|
|
|
scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py")) |
|
execute([ |
|
"python", |
|
"-B", # no .py[co] |
|
scriptPath, |
|
"--name=%s" % packageBuildInfo.getReleaseVersion(), |
|
"--id=0x%08x" % packageBuildInfo.getReleaseId(), |
|
"--out=%s" % self.dstFile |
|
]) |
|
|
|
class GenCMake (DstFile): |
|
def __init__ (self, srcFile, dstFile, replaceVars): |
|
super(GenCMake, self).__init__(dstFile) |
|
self.srcFile = srcFile |
|
self.replaceVars = replaceVars |
|
|
|
def make (self, packageBuildInfo): |
|
self.makeDir() |
|
print " GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath) |
|
src = readFile(self.srcFile) |
|
for var, value in self.replaceVars: |
|
src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var), |
|
'set(%s "%s"' % (var, value), src) |
|
writeFile(self.dstFile, src) |
|
|
|
def createFileTargets (srcBasePath, dstBasePath, files, filters): |
|
usedFiles = set() # Files that are already included by other filters |
|
targets = [] |
|
|
|
for isMatch, createFileObj in filters: |
|
# Build list of files that match filter |
|
matchingFiles = [] |
|
for file in files: |
|
if not file in usedFiles and isMatch(file): |
|
matchingFiles.append(file) |
|
|
|
# Build file objects, add to used set |
|
for file in matchingFiles: |
|
usedFiles.add(file) |
|
targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file))) |
|
|
|
return targets |
|
|
|
# Generates multiple file targets based on filters |
|
class FileTargetGroup: |
|
def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath): |
|
self.srcBasePath = srcBasePath |
|
self.dstBasePath = dstBasePath |
|
self.filters = filters |
|
self.getSrcBasePath = srcBasePathFunc |
|
|
|
def make (self, packageBuildInfo): |
|
fullSrcPath = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath)) |
|
fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath)) |
|
|
|
allFiles = getFileList(fullSrcPath) |
|
targets = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters) |
|
|
|
# Make all file targets |
|
for file in targets: |
|
file.make(packageBuildInfo) |
|
|
|
# Single file target |
|
class SingleFileTarget: |
|
def __init__ (self, srcFile, dstFile, makeTarget): |
|
self.srcFile = srcFile |
|
self.dstFile = dstFile |
|
self.makeTarget = makeTarget |
|
|
|
def make (self, packageBuildInfo): |
|
fullSrcPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile)) |
|
fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile)) |
|
|
|
target = self.makeTarget(fullSrcPath, fullDstPath) |
|
target.make(packageBuildInfo) |
|
|
|
class BuildTarget: |
|
def __init__ (self, baseConfig, generator, targets = None): |
|
self.baseConfig = baseConfig |
|
self.generator = generator |
|
self.targets = targets |
|
|
|
def make (self, packageBuildInfo): |
|
print " Building %s" % self.baseConfig.getBuildDir() |
|
|
|
# Create config with full build dir path |
|
config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()), |
|
self.baseConfig.getBuildType(), |
|
self.baseConfig.getArgs(), |
|
srcPath = os.path.join(packageBuildInfo.dstBasePath, "src")) |
|
|
|
assert not os.path.exists(config.getBuildDir()) |
|
build(config, self.generator, self.targets) |
|
|
|
class BuildAndroidTarget: |
|
def __init__ (self, dstFile): |
|
self.dstFile = dstFile |
|
|
|
def make (self, packageBuildInfo): |
|
print " Building Android binary" |
|
|
|
buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build") |
|
|
|
assert not os.path.exists(buildRoot) |
|
os.makedirs(buildRoot) |
|
|
|
# Execute build script |
|
scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py")) |
|
execute([ |
|
"python", |
|
"-B", # no .py[co] |
|
scriptPath, |
|
"--build-root=%s" % buildRoot, |
|
]) |
|
|
|
srcFile = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk")) |
|
dstFile = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile)) |
|
|
|
CopyFile(srcFile, dstFile).make(packageBuildInfo) |
|
|
|
class FetchExternalSourcesTarget: |
|
def __init__ (self): |
|
pass |
|
|
|
def make (self, packageBuildInfo): |
|
scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py")) |
|
execute([ |
|
"python", |
|
"-B", # no .py[co] |
|
scriptPath, |
|
]) |
|
|
|
class RemoveSourcesTarget: |
|
def __init__ (self): |
|
pass |
|
|
|
def make (self, packageBuildInfo): |
|
shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False) |
|
|
|
class Module: |
|
def __init__ (self, name, targets): |
|
self.name = name |
|
self.targets = targets |
|
|
|
def make (self, packageBuildInfo): |
|
for target in self.targets: |
|
target.make(packageBuildInfo) |
|
|
|
class ReleaseConfig: |
|
def __init__ (self, name, version, modules, sources = True): |
|
self.name = name |
|
self.version = version |
|
self.modules = modules |
|
self.sources = sources |
|
|
|
def getName (self): |
|
return self.name |
|
|
|
def getVersion (self): |
|
return self.version |
|
|
|
def getModules (self): |
|
return self.modules |
|
|
|
def packageWithSources (self): |
|
return self.sources |
|
|
|
def matchIncludeExclude (includePatterns, excludePatterns, filename): |
|
components = os.path.normpath(filename).split(os.sep) |
|
for pattern in excludePatterns: |
|
for component in components: |
|
if fnmatch.fnmatch(component, pattern): |
|
return False |
|
|
|
for pattern in includePatterns: |
|
for component in components: |
|
if fnmatch.fnmatch(component, pattern): |
|
return True |
|
|
|
return False |
|
|
|
def copyFileFilter (includePatterns, excludePatterns=[]): |
|
return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f), |
|
lambda s, d: CopyFile(s, d)) |
|
|
|
def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]): |
|
return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)]) |
|
|
|
def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]): |
|
return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath) |
|
|
|
def makeFileCopy (srcFile, dstFile): |
|
return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d)) |
|
|
|
def getReleaseFileName (configName, releaseName): |
|
today = datetime.date.today() |
|
return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName) |
|
|
|
def getTempDir (): |
|
dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases") |
|
if not os.path.exists(dirName): |
|
os.makedirs(dirName) |
|
return dirName |
|
|
|
def makeRelease (releaseConfig): |
|
releaseName = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion()) |
|
tmpPath = getTempDir() |
|
srcBasePath = DEQP_DIR |
|
dstBasePath = os.path.join(tmpPath, releaseName) |
|
tmpBasePath = os.path.join(tmpPath, releaseName + "-tmp") |
|
packageBuildInfo = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath) |
|
dstArchiveName = releaseName + ".tar.bz2" |
|
|
|
print "Creating release %s to %s" % (releaseName, tmpPath) |
|
|
|
# Remove old temporary dirs |
|
for path in [dstBasePath, tmpBasePath]: |
|
if os.path.exists(path): |
|
shutil.rmtree(path, ignore_errors=False) |
|
|
|
# Make all modules |
|
for module in releaseConfig.getModules(): |
|
print " Processing module %s" % module.name |
|
module.make(packageBuildInfo) |
|
|
|
# Remove sources? |
|
if not releaseConfig.packageWithSources(): |
|
shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False) |
|
|
|
# Create archive |
|
print "Creating %s" % dstArchiveName |
|
archive = tarfile.open(dstArchiveName, 'w:bz2') |
|
archive.add(dstBasePath, arcname=releaseName) |
|
archive.close() |
|
|
|
# Remove tmp dirs |
|
for path in [dstBasePath, tmpBasePath]: |
|
if os.path.exists(path): |
|
shutil.rmtree(path, ignore_errors=False) |
|
|
|
print "Done!" |
|
|
|
# Module declarations |
|
|
|
SRC_FILE_PATTERNS = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"] |
|
TARGET_PATTERNS = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"] |
|
|
|
BASE = Module("Base", [ |
|
makeFileCopy ("LICENSE", "src/LICENSE"), |
|
makeFileCopy ("CMakeLists.txt", "src/CMakeLists.txt"), |
|
makeFileCopyGroup ("targets", "src/targets", TARGET_PATTERNS), |
|
makeFileCopyGroup ("execserver", "src/execserver", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("executor", "src/executor", SRC_FILE_PATTERNS), |
|
makeFileCopy ("modules/CMakeLists.txt", "src/modules/CMakeLists.txt"), |
|
makeFileCopyGroup ("external", "src/external", ["CMakeLists.txt", "*.py"]), |
|
|
|
# Stylesheet for displaying test logs on browser |
|
makeFileCopyGroup ("doc/testlog-stylesheet", "doc/testlog-stylesheet", ["*"]), |
|
|
|
# Non-optional parts of framework |
|
makeFileCopy ("framework/CMakeLists.txt", "src/framework/CMakeLists.txt"), |
|
makeFileCopyGroup ("framework/delibs", "src/framework/delibs", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("framework/common", "src/framework/common", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("framework/qphelper", "src/framework/qphelper", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("framework/platform", "src/framework/platform", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("framework/opengl", "src/framework/opengl", SRC_FILE_PATTERNS, ["simplereference"]), |
|
makeFileCopyGroup ("framework/egl", "src/framework/egl", SRC_FILE_PATTERNS), |
|
|
|
# android sources |
|
makeFileCopyGroup ("android/package/src", "src/android/package/src", SRC_FILE_PATTERNS), |
|
makeFileCopy ("android/package/AndroidManifest.xml", "src/android/package/AndroidManifest.xml"), |
|
makeFileCopyGroup ("android/package/res", "src/android/package/res", ["*.png", "*.xml"]), |
|
makeFileCopyGroup ("android/scripts", "src/android/scripts", [ |
|
"common.py", |
|
"build.py", |
|
"resources.py", |
|
"install.py", |
|
"launch.py", |
|
"debug.py" |
|
]), |
|
|
|
# Release info |
|
GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl") |
|
]) |
|
|
|
DOCUMENTATION = Module("Documentation", [ |
|
makeFileCopyGroup ("doc/pdf", "doc", ["*.pdf"]), |
|
makeFileCopyGroup ("doc", "doc", ["porting_layer_changes_*.txt"]), |
|
]) |
|
|
|
GLSHARED = Module("Shared GL Tests", [ |
|
# Optional framework components |
|
makeFileCopyGroup ("framework/randomshaders", "src/framework/randomshaders", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("framework/opengl/simplereference", "src/framework/opengl/simplereference", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("framework/referencerenderer", "src/framework/referencerenderer", SRC_FILE_PATTERNS), |
|
|
|
makeFileCopyGroup ("modules/glshared", "src/modules/glshared", SRC_FILE_PATTERNS), |
|
]) |
|
|
|
GLES2 = Module("GLES2", [ |
|
makeFileCopyGroup ("modules/gles2", "src/modules/gles2", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("data/gles2", "src/data/gles2", ["*.*"]), |
|
makeFileCopyGroup ("doc/testspecs/GLES2", "doc/testspecs/GLES2", ["*.txt"]) |
|
]) |
|
|
|
GLES3 = Module("GLES3", [ |
|
makeFileCopyGroup ("modules/gles3", "src/modules/gles3", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("data/gles3", "src/data/gles3", ["*.*"]), |
|
makeFileCopyGroup ("doc/testspecs/GLES3", "doc/testspecs/GLES3", ["*.txt"]) |
|
]) |
|
|
|
GLES31 = Module("GLES31", [ |
|
makeFileCopyGroup ("modules/gles31", "src/modules/gles31", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("data/gles31", "src/data/gles31", ["*.*"]), |
|
makeFileCopyGroup ("doc/testspecs/GLES31", "doc/testspecs/GLES31", ["*.txt"]) |
|
]) |
|
|
|
EGL = Module("EGL", [ |
|
makeFileCopyGroup ("modules/egl", "src/modules/egl", SRC_FILE_PATTERNS) |
|
]) |
|
|
|
INTERNAL = Module("Internal", [ |
|
makeFileCopyGroup ("modules/internal", "src/modules/internal", SRC_FILE_PATTERNS), |
|
makeFileCopyGroup ("data/internal", "src/data/internal", ["*.*"]), |
|
]) |
|
|
|
EXTERNAL_SRCS = Module("External sources", [ |
|
FetchExternalSourcesTarget() |
|
]) |
|
|
|
ANDROID_BINARIES = Module("Android Binaries", [ |
|
BuildAndroidTarget ("bin/android/dEQP.apk"), |
|
makeFileCopyGroup ("targets/android", "bin/android", ["*.bat", "*.sh"]), |
|
]) |
|
|
|
COMMON_BUILD_ARGS = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))] |
|
NULL_X32_CONFIG = BuildConfig('null-x32', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS) |
|
NULL_X64_CONFIG = BuildConfig('null-x64', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS) |
|
GLX_X32_CONFIG = BuildConfig('glx-x32', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS) |
|
GLX_X64_CONFIG = BuildConfig('glx-x64', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS) |
|
|
|
EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"] |
|
|
|
LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [ |
|
BuildTarget (NULL_X32_CONFIG, ANY_UNIX_GENERATOR), |
|
makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES), |
|
makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES), |
|
]) |
|
|
|
LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [ |
|
BuildTarget (NULL_X64_CONFIG, ANY_UNIX_GENERATOR), |
|
makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES), |
|
makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES), |
|
]) |
|
|
|
# Special module to remove src dir, for example after binary build |
|
REMOVE_SOURCES = Module("Remove sources from package", [ |
|
RemoveSourcesTarget() |
|
]) |
|
|
|
# Release configuration |
|
|
|
ALL_MODULES = [ |
|
BASE, |
|
DOCUMENTATION, |
|
GLSHARED, |
|
GLES2, |
|
GLES3, |
|
GLES31, |
|
EGL, |
|
INTERNAL, |
|
EXTERNAL_SRCS, |
|
] |
|
|
|
ALL_BINARIES = [ |
|
LINUX_X64_COMMON_BINARIES, |
|
ANDROID_BINARIES, |
|
] |
|
|
|
RELEASE_CONFIGS = { |
|
"src": ALL_MODULES, |
|
"src-bin": ALL_MODULES + ALL_BINARIES, |
|
"bin": ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES], |
|
} |
|
|
|
def parseArgs (): |
|
parser = argparse.ArgumentParser(description = "Build release package") |
|
parser.add_argument("-c", |
|
"--config", |
|
dest="config", |
|
choices=RELEASE_CONFIGS.keys(), |
|
required=True, |
|
help="Release configuration") |
|
parser.add_argument("-n", |
|
"--name", |
|
dest="name", |
|
required=True, |
|
help="Package-specific name") |
|
parser.add_argument("-v", |
|
"--version", |
|
dest="version", |
|
required=True, |
|
help="Version code") |
|
return parser.parse_args() |
|
|
|
if __name__ == "__main__": |
|
args = parseArgs() |
|
config = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config]) |
|
makeRelease(config)
|
|
|