diff --git a/Docs/Whats_New.md b/Docs/Whats_New.md index fc72286a..9fdf2d32 100644 --- a/Docs/Whats_New.md +++ b/Docs/Whats_New.md @@ -29,6 +29,7 @@ Released TBD - Support `libMoltenVK.dylib` for _iOS Simulator_ architecture. - Restore support for _iOS Simulator_ destination in recent update to _Cube_ demo that uses dynamic-linking. - Don't update `currentExtent` of headless surface when swapchain attached. +- `runcts` script also output a file containing a list of the failed CTS tests. diff --git a/MoltenVKPackaging.xcodeproj/project.pbxproj b/MoltenVKPackaging.xcodeproj/project.pbxproj index 43d05866..d503399e 100644 --- a/MoltenVKPackaging.xcodeproj/project.pbxproj +++ b/MoltenVKPackaging.xcodeproj/project.pbxproj @@ -232,6 +232,7 @@ A9DDBF8C26827F02005DD991 /* runcts */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = runcts; sourceTree = ""; }; A9FC5F60249D2ED3003CB086 /* package_all.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = package_all.sh; sourceTree = ""; }; A9FC5F64249D3778003CB086 /* create_dylib_tvos.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = create_dylib_tvos.sh; sourceTree = ""; }; + A9FE05632B697AE400208814 /* get_failing_cts_tests.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = get_failing_cts_tests.py; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXGroup section */ @@ -280,6 +281,7 @@ A975D55C213F25D700D4834F /* create_dylib.sh */, A9B1007A24F837AF00EADC6E /* create_xcframework_func.sh */, A980A25F24C628F3007A8F6F /* gen_moltenvk_rev_hdr.sh */, + A9FE05632B697AE400208814 /* get_failing_cts_tests.py */, A9FC5F60249D2ED3003CB086 /* package_all.sh */, A92EF7C921856EA200C8B91B /* package_docs.sh */, A9CBBFF924F8A1EB006D41EF /* package_moltenvk_xcframework.sh */, diff --git a/Scripts/get_failing_cts_tests.py b/Scripts/get_failing_cts_tests.py new file mode 100644 index 00000000..0f970c77 --- /dev/null +++ b/Scripts/get_failing_cts_tests.py @@ -0,0 +1,36 @@ +import argparse + +def getArguments(): + parser = argparse.ArgumentParser(description="Retrieve failing tests from CTS output file") + parser.add_argument("inputFile", help="Input file") + parser.add_argument("-o", "--output", dest="outputFile", default="failingTests.txt", type=str, required=False, help="Output file to store failing tests") + parser.add_argument('-q', action='store_true') + return parser.parse_args() + +if __name__ == "__main__": + args = getArguments() + + with open(args.inputFile, "r") as inputFile: + with open(args.outputFile, "w") as outputFile: + if not args.q: + print("Reading file " + args.inputFile + " for test failures") + + testName = "" + for line in inputFile: + # Empty lines means we have changed test + if line == "\n": + testName = "" + continue + + # Assuming lines with test names are formatted like: "Test case 'dEQP-VK.test.name'.." + if "Test case" in line: + testName = line[11:-4] + continue + + # Failed tests will have a line after the test that should be like: "Fail (sometimes the reason here)" + if testName != "" and "Fail" in line: + outputFile.writelines(testName + "\n") + + if not args.q: + print("Failures written to " + args.outputFile) + pass diff --git a/Scripts/runcts b/Scripts/runcts index 16d33685..6b94317d 100755 --- a/Scripts/runcts +++ b/Scripts/runcts @@ -25,6 +25,11 @@ # case_list_file, with "-results.txt" appended. CTS will also output several # working temporary files into the directory holding this script. # +# -fo path +# The path to the file to write a list of the failed CTS tests. If this option +# is not provided, the list of failed tests will be written to an output text +# file with the same name as the case_list_file, with "-fails.txt" appended. +# # --cts path # The path to the directory containing the built CTS executable. # If this parameter is not provided, it defaults to: @@ -44,6 +49,7 @@ cts_vk_dir="../../VK-GL-CTS/build/external/vulkancts/modules/vulkan/Debug" caselist_file="" results_file="" +failures_file="" is_portability="" while (( "$#" )); do @@ -52,6 +58,10 @@ while (( "$#" )); do results_file="${2}" shift 2 ;; + -fo) + failures_file="${2}" + shift 2 + ;; --cts) cts_vk_dir="${2}" shift 2 @@ -80,6 +90,10 @@ if [ "${results_file}" == "" ]; then results_file="${caselist_file}-results.txt" fi +if [ "${failures_file}" == "" ]; then + failures_file="${caselist_file}-fails.txt" +fi + # -------------- MoltenVK configuration -------------------- @@ -131,4 +145,6 @@ start_time=${SECONDS} --deqp-caselist-file="${caselist_file}" \ &> "${results_file}" +python3 ./get_failing_cts_tests.py -q -o "${failures_file}" "${results_file}" + echo Testing complete in $(($SECONDS - $start_time)) seconds at `date +%r`