Merge pull request #2147 from billhollings/runcts-output-failed-tests

runcts script also output a file containing a list of the failed CTS tests.

Docs/Whats_New.md

@@ -29,6 +29,7 @@ Released TBD
 - Support `libMoltenVK.dylib` for _iOS Simulator_ architecture.
 - Restore support for _iOS Simulator_ destination in recent update to _Cube_ demo that uses dynamic-linking.
 - Don't update `currentExtent` of headless surface when swapchain attached.
+- `runcts` script also output a file containing a list of the failed CTS tests.
 
 
 

MoltenVKPackaging.xcodeproj/project.pbxproj

@@ -232,6 +232,7 @@
 		A9DDBF8C26827F02005DD991 /* runcts */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = runcts; sourceTree = "<group>"; };
 		A9FC5F60249D2ED3003CB086 /* package_all.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = package_all.sh; sourceTree = "<group>"; };
 		A9FC5F64249D3778003CB086 /* create_dylib_tvos.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = create_dylib_tvos.sh; sourceTree = "<group>"; };
+		A9FE05632B697AE400208814 /* get_failing_cts_tests.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = get_failing_cts_tests.py; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXGroup section */
@@ -280,6 +281,7 @@
 				A975D55C213F25D700D4834F /* create_dylib.sh */,
 				A9B1007A24F837AF00EADC6E /* create_xcframework_func.sh */,
 				A980A25F24C628F3007A8F6F /* gen_moltenvk_rev_hdr.sh */,
+				A9FE05632B697AE400208814 /* get_failing_cts_tests.py */,
 				A9FC5F60249D2ED3003CB086 /* package_all.sh */,
 				A92EF7C921856EA200C8B91B /* package_docs.sh */,
 				A9CBBFF924F8A1EB006D41EF /* package_moltenvk_xcframework.sh */,

Scripts/get_failing_cts_tests.py

@@ -0,0 +1,36 @@
+import argparse
+
+def getArguments():
+    parser = argparse.ArgumentParser(description="Retrieve failing tests from CTS output file")
+    parser.add_argument("inputFile", help="Input file")
+    parser.add_argument("-o", "--output", dest="outputFile", default="failingTests.txt", type=str, required=False, help="Output file to store failing tests")
+    parser.add_argument('-q', action='store_true')
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    args = getArguments()
+
+    with open(args.inputFile, "r") as inputFile:
+        with open(args.outputFile, "w") as outputFile:
+            if not args.q:
+                print("Reading file " + args.inputFile + " for test failures")
+
+            testName = ""
+            for line in inputFile:
+                # Empty lines means we have changed test
+                if line == "\n":
+                    testName = ""
+                    continue
+
+                # Assuming lines with test names are formatted like: "Test case 'dEQP-VK.test.name'.."
+                if "Test case" in line:
+                    testName = line[11:-4]
+                    continue
+
+                # Failed tests will have a line after the test that should be like: "Fail (sometimes the reason here)"
+                if testName != "" and "Fail" in line:
+                    outputFile.writelines(testName + "\n")
+
+            if not args.q:
+                print("Failures written to " + args.outputFile)
+    pass

Scripts/runcts

@@ -25,6 +25,11 @@
 #         case_list_file, with "-results.txt" appended. CTS will also output several
 #         working temporary files into the directory holding this script.
 #
+#     -fo path
+#         The path to the file to write a list of the failed CTS tests. If this option
+#         is not provided, the list of failed tests will be written to an output text
+#         file with the same name as the case_list_file, with "-fails.txt" appended.
+#
 #     --cts path
 #         The path to the directory containing the built CTS executable.
 #         If this parameter is not provided, it defaults to:
@@ -44,6 +49,7 @@
 cts_vk_dir="../../VK-GL-CTS/build/external/vulkancts/modules/vulkan/Debug"
 caselist_file=""
 results_file=""
+failures_file=""
 is_portability=""
 
 while (( "$#" )); do
@@ -52,6 +58,10 @@ while (( "$#" )); do
          results_file="${2}"
          shift 2
          ;;
+       -fo)
+         failures_file="${2}"
+         shift 2
+         ;;
        --cts)
          cts_vk_dir="${2}"
          shift 2
@@ -80,6 +90,10 @@ if [ "${results_file}" == "" ]; then
 	results_file="${caselist_file}-results.txt"
 fi
 
+if [ "${failures_file}" == "" ]; then
+	failures_file="${caselist_file}-fails.txt"
+fi
+
 
 # -------------- MoltenVK configuration --------------------
 
@@ -131,4 +145,6 @@ start_time=${SECONDS}
 --deqp-caselist-file="${caselist_file}"    \
 &> "${results_file}"
 
+python3 ./get_failing_cts_tests.py -q -o "${failures_file}" "${results_file}"
+
 echo Testing complete in $(($SECONDS - $start_time)) seconds at `date +%r`