From 25d1349579d026d31bf6c568d8ecf75bfe7a325a Mon Sep 17 00:00:00 2001 From: Bill Hollings Date: Tue, 22 Jun 2021 19:45:24 -0400 Subject: [PATCH] Add Scripts/runcts script as a convenience for running Vulkan CTS tests. Update .gitignore to ignore CTS artifacts in Scripts directory. --- .gitignore | 4 + Docs/Whats_New.md | 1 + MoltenVKPackaging.xcodeproj/project.pbxproj | 2 + Scripts/runcts | 124 ++++++++++++++++++++ 4 files changed, 131 insertions(+) create mode 100755 Scripts/runcts diff --git a/.gitignore b/.gitignore index b275fef2..8a10068e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ External/Vulkan-Portability External/Vulkan-Tools External/VulkanSamples +# Vulkan CTS testing artifacts +Scripts/TestResults.qpa +Scripts/shadercache.bin + # Mac OS X Finder .DS_Store diff --git a/Docs/Whats_New.md b/Docs/Whats_New.md index 6ae284d2..f5c70752 100644 --- a/Docs/Whats_New.md +++ b/Docs/Whats_New.md @@ -39,6 +39,7 @@ Released TBD - Protect against crash when retrieving `MTLTexture` when `VkImage` has no `VkDeviceMemory` bound. - Adjust some `VkPhysicalDeviceLimits` values for Vulkan and Metal compliance. - Fix internal reference from `SPIRV_CROSS_NAMESPACE_OVERRIDE` to `SPIRV_CROSS_NAMESPACE`. +- Add `Scripts/runcts` script as a convenience for running Vulkan CTS tests. - Support _Xcode 13_ SDK APIs and build settings. diff --git a/MoltenVKPackaging.xcodeproj/project.pbxproj b/MoltenVKPackaging.xcodeproj/project.pbxproj index 4a92ac47..c617396e 100644 --- a/MoltenVKPackaging.xcodeproj/project.pbxproj +++ b/MoltenVKPackaging.xcodeproj/project.pbxproj @@ -195,6 +195,7 @@ A9C70F45221B04C800FBA31A /* create_dylib_ios.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = create_dylib_ios.sh; sourceTree = ""; }; A9CBBFF924F8A1EB006D41EF /* package_moltenvk_xcframework.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = package_moltenvk_xcframework.sh; sourceTree = ""; }; A9DA8341218A198C002AA662 /* .gitignore */ = {isa = PBXFileReference; lastKnownFileType = text; path = .gitignore; sourceTree = ""; }; + A9DDBF8C26827F02005DD991 /* runcts */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = runcts; sourceTree = ""; }; A9FC5F60249D2ED3003CB086 /* package_all.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = package_all.sh; sourceTree = ""; }; A9FC5F64249D3778003CB086 /* create_dylib_tvos.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = create_dylib_tvos.sh; sourceTree = ""; }; /* End PBXFileReference section */ @@ -250,6 +251,7 @@ A92EF7C721856EA200C8B91B /* package_shader_converter_tool.sh */, A92EF7CB21856EA300C8B91B /* package_shader_converter_xcframework.sh */, A92EF7CA21856EA200C8B91B /* package_update_latest.sh */, + A9DDBF8C26827F02005DD991 /* runcts */, A92EF7DE2186451700C8B91B /* Makefile */, ); path = Scripts; diff --git a/Scripts/runcts b/Scripts/runcts new file mode 100755 index 00000000..40c01645 --- /dev/null +++ b/Scripts/runcts @@ -0,0 +1,124 @@ +#!/bin/bash + +# Copyright (c) 2016-2021 The Brenwill Workshop Ltd. +# +# runcts - This script simplifies running Vulkan CTS tests on MoltenVK. +# +# CTS will save temporary output files to the directory this script is located in. It is +# recommended that you copy this file to a location where your testing files will be kept. +# By default, this should be a directory beside the VK-GL-CTS directory. See the description +# of the --cts option below for information about locations. +# +# Executing a large case list file of CTS tests can take a long time. It is recommended +# that you run this using caffeinate, as follows: +# caffeinate -is ./runcts [-o path] [--cts path] [-p] [--portability] case_list_file +# +# +# macOS usage: ./runcts [-o path] [--cts path] [-p] [--portability] case_list_file +# +# case_list_file +# The path to the file that contains a list of the CTS test cases to run. +# +# -o path +# The path to the file to write the test results. If this option is not provided, +# the results will be written to an output text file with the same name as the +# case_list_file, with "-results.txt" appended. CTS will also output several +# working temporary files into the directory holding this script. +# +# --cts path +# The path to the directory containing the built CTS executable. +# If this parameter is not provided, it defaults to: +# "../../VK-GL-CTS/build/external/vulkancts/modules/vulkan/Debug". +# +# -p +# Same as the --portability option. +# +# --portability +# Indicates that this testing is for testing conformance with the +# Vulkan Portability initiative. MoltenVK will advertise support for +# only Vulkan 1.0, and only the following extensions: +# VK_KHR_get_physical_device_properties2 +# VK_KHR_portability_subset +# VK_MVK_moltenvk +# + +cts_vk_dir="../../VK-GL-CTS/build/external/vulkancts/modules/vulkan/Debug" +caselist_file="" +results_file="" +is_portability="" + +while (( "$#" )); do + case "$1" in + -o) + results_file="${2}" + shift 2 + ;; + --cts) + cts_vk_dir="${2}" + shift 2 + ;; + -p) + is_portability="Y" + shift 1 + ;; + --portability) + is_portability="Y" + shift 1 + ;; + -*|--*=) + echo "Error: Unsupported option $1. See usage instructions in body of this script." >&2 + exit 1 + ;; + *) + caselist_file="${1}" + shift 1 + ;; + esac +done + +if [ "${caselist_file}" == "" ]; then + echo "Error: No caselist file specified. See usage instructions in body of this script." >&2 + exit 1 +fi + +if [ "${results_file}" == "" ]; then + results_file="${caselist_file}-results.txt" +fi + + +# -------------- MoltenVK configuration -------------------- + +if [ "${is_portability}" != "" ]; then + export MVK_CONFIG_API_VERSION_TO_ADVERTISE=4194304 + export MVK_CONFIG_ADVERTISE_EXTENSIONS=0xA +fi + +export METAL_DEVICE_WRAPPER_TYPE=1 +export METAL_ERROR_MODE=3 +export METAL_DEBUG_ERROR_MODE=3 + +# ----- MoltenVK config settings ------ +export MVK_CONFIG_LOG_LEVEL=0 +export MVK_DEBUG=0 + +# Additional MoltenVK configuration can be set here by +# editing below, or can be set before calling this script. +export MVK_CONFIG_RESUME_LOST_DEVICE=1 +export MVK_CONFIG_FAST_MATH_ENABLED=0 +export MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS=0 +export MVK_CONFIG_FORCE_LOW_POWER_GPU=0 + + +# -------------- Operation -------------------- + +echo Testing started at `date +%r` +start_time=${SECONDS} + +"${cts_vk_dir}/deqp-vk" \ +--deqp-archive-dir="${cts_vk_dir}/.." \ +--deqp-log-images=disable \ +--deqp-log-shader-sources=disable \ +--deqp-caselist-file="${caselist_file}" \ +&> "${results_file}" + +echo Testing complete in $(($SECONDS - $start_time)) seconds