From e6a8409b31c72e2929f2dbd9ef8f4f83841e9cb4 Mon Sep 17 00:00:00 2001
From: Chip Davis <cdavis@codeweavers.com>
Date: Wed, 9 Dec 2020 01:37:14 -0600
Subject: [PATCH] MVKGraphicsPipeline: Fix color write mask with RGB9E5 RTs.

Metal does not allow color write masks with this format where some but
not all of the channels are disabled. Either all must be enabled or none
must be enabled. Presumably, this is because of the shared exponent.

This is just good enough to stop the validation layer from violently
terminating the program. To implement this properly requires using
framebuffer fetch, with a change to SPIRV-Cross. Luckily, the only GPUs
that support `RGB9E5` rendering also support framebuffer fetch.
Honestly, I don't understand why Apple's drivers don't do this.
---
 MoltenVK/MoltenVK/GPUObjects/MVKPipeline.mm | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/MoltenVK/MoltenVK/GPUObjects/MVKPipeline.mm b/MoltenVK/MoltenVK/GPUObjects/MVKPipeline.mm
index 7a160fa2..6fb4ec12 100644
--- a/MoltenVK/MoltenVK/GPUObjects/MVKPipeline.mm
+++ b/MoltenVK/MoltenVK/GPUObjects/MVKPipeline.mm
@@ -1387,7 +1387,15 @@ void MVKGraphicsPipeline::addFragmentOutputToPipeline(MTLRenderPipelineDescripto
 
             MTLRenderPipelineColorAttachmentDescriptor* colorDesc = plDesc.colorAttachments[caIdx];
             colorDesc.pixelFormat = getPixelFormats()->getMTLPixelFormat(mvkRenderSubpass->getColorAttachmentFormat(caIdx));
-            colorDesc.writeMask = mvkMTLColorWriteMaskFromVkChannelFlags(pCA->colorWriteMask);
+            if (colorDesc.pixelFormat == MTLPixelFormatRGB9E5Float) {
+                // Metal doesn't allow disabling individual channels for a RGB9E5 render target.
+                // Either all must be disabled or none must be disabled.
+                // TODO: Use framebuffer fetch to support this anyway. I don't understand why Apple doesn't
+                // support it, given that the only GPUs that support this in Metal also support framebuffer fetch.
+                colorDesc.writeMask = pCA->colorWriteMask ? MTLColorWriteMaskAll : MTLColorWriteMaskNone;
+            } else {
+                colorDesc.writeMask = mvkMTLColorWriteMaskFromVkChannelFlags(pCA->colorWriteMask);
+            }
             // Don't set the blend state if we're not using this attachment.
             // The pixel format will be MTLPixelFormatInvalid in that case, and
             // Metal asserts if we turn on blending with that pixel format.