From 8e6731fd8e73443852a700e22431c224b37e2021 Mon Sep 17 00:00:00 2001 From: Bill Hollings Date: Tue, 10 Aug 2021 10:29:09 -0400 Subject: [PATCH 1/2] Revert to prefer MTLEvent over MTLFence for VkSemaphore, except on NVIDIA. Prefer MTLEvent over MTLFence for VkSemaphore, because MTLEvent handles sync across MTLCommandBuffers and MTLCommandQueues, except on NVIDIA GPUs, which have demonstrated trouble with MTLEvents, prefer MTLFence. Add MVKDevice::VkSemaphoreStyle enum. --- Docs/Whats_New.md | 1 + MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h | 26 +++++++------- MoltenVK/MoltenVK/GPUObjects/MVKDevice.h | 10 ++++-- MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm | 42 +++++++++++++++++------ 4 files changed, 54 insertions(+), 25 deletions(-) diff --git a/Docs/Whats_New.md b/Docs/Whats_New.md index 899f6217..b16a39b7 100644 --- a/Docs/Whats_New.md +++ b/Docs/Whats_New.md @@ -18,6 +18,7 @@ MoltenVK 1.1.5 Released TBD +- Revert to prefer `MTLEvent` over `MTLFence` for `VkSemaphore`, except on NVIDIA. - Vulkan timestamp query pools use Metal GPU counters when available. - Support resolving attachments with formats that Metal does not natively resolve. - Fix issue where swapchain images were acquired out of order under heavy load. diff --git a/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h b/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h index d9a6820d..99138dfe 100644 --- a/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h +++ b/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h @@ -557,14 +557,15 @@ typedef struct { /** * Use MTLFence, if it is available on the device, for VkSemaphore synchronization behaviour. * - * This parameter interacts with semaphoreUseMTLEvent. If both are enabled, semaphoreUseMTLFence - * takes priority and MTLFence will be used if it is available, otherwise MTLEvent will be used - * if it is available. If neither semaphoreUseMTLFence or semaphoreUseMTLEvent are enabled, or - * if neither MTLFence or MTLEvent are available, CPU-based synchoronization will be used. + * This parameter interacts with semaphoreUseMTLEvent. If both are enabled, on GPUs other than + * NVIDIA, semaphoreUseMTLEvent takes priority and MTLEvent will be used if it is available, + * otherwise MTLFence will be used if it is available. On NVIDIA GPUs, the opposite priority + * applies. If neither semaphoreUseMTLFence nor semaphoreUseMTLEvent are enabled, or if neither + * MTLEvent nor MTLFence are available, CPU-based synchoronization will be used. * * In the special case of VK_SEMAPHORE_TYPE_TIMELINE semaphores, MoltenVK will always * use MTLSharedEvent if it is available on the platform, regardless of the values of - * MVK_ALLOW_METAL_FENCES or MVK_ALLOW_METAL_EVENTS. + * semaphoreUseMTLEvent or semaphoreUseMTLFence. * * The value of this parameter must be changed before creating a VkDevice, * for the change to take effect. @@ -573,21 +574,22 @@ typedef struct { * MVK_ALLOW_METAL_FENCES * runtime environment variable or MoltenVK compile-time build setting. * If neither is set, this setting is enabled by default, and VkSemaphore will use MTLFence, - * if it is available. + * if it is available, unless MTLEvent is available and semaphoreUseMTLEvent is enabled. */ VkBool32 semaphoreUseMTLFence; /** * Use MTLEvent, if it is available on the device, for VkSemaphore synchronization behaviour. * - * This parameter interacts with semaphoreUseMTLFence. If both are enabled, semaphoreUseMTLFence - * takes priority and MTLFence will be used if it is available, otherwise MTLEvent will be used - * if it is available. If neither semaphoreUseMTLFence or semaphoreUseMTLEvent are enabled, or - * if neither MTLFence or MTLEvent are available, CPU-based synchoronization will be used. + * This parameter interacts with semaphoreUseMTLFence. If both are enabled, on GPUs other than + * NVIDIA, semaphoreUseMTLEvent takes priority and MTLEvent will be used if it is available, + * otherwise MTLFence will be used if it is available. On NVIDIA GPUs, the opposite priority + * applies. If neither semaphoreUseMTLFence nor semaphoreUseMTLEvent are enabled, or if neither + * MTLEvent nor MTLFence are available, CPU-based synchoronization will be used. * * In the special case of VK_SEMAPHORE_TYPE_TIMELINE semaphores, MoltenVK will always * use MTLSharedEvent if it is available on the platform, regardless of the values of - * MVK_ALLOW_METAL_FENCES or MVK_ALLOW_METAL_EVENTS. + * semaphoreUseMTLEvent or semaphoreUseMTLFence. * * The value of this parameter must be changed before creating a VkDevice, * for the change to take effect. @@ -596,7 +598,7 @@ typedef struct { * MVK_ALLOW_METAL_EVENTS * runtime environment variable or MoltenVK compile-time build setting. * If neither is set, this setting is enabled by default, and VkSemaphore will use MTLEvent, - * if it is available, unless if MTLFence is available and semaphoreUseMTLFence is enabled. + * if it is available. */ VkBool32 semaphoreUseMTLEvent; diff --git a/MoltenVK/MoltenVK/GPUObjects/MVKDevice.h b/MoltenVK/MoltenVK/GPUObjects/MVKDevice.h index d4d65d73..11baf05a 100644 --- a/MoltenVK/MoltenVK/GPUObjects/MVKDevice.h +++ b/MoltenVK/MoltenVK/GPUObjects/MVKDevice.h @@ -832,11 +832,17 @@ protected: id _dummyBlitMTLBuffer; uint32_t _globalVisibilityQueryCount; std::mutex _vizLock; - bool _useMTLFenceForSemaphores; - bool _useMTLEventForSemaphores; bool _logActivityPerformanceInline; bool _isPerformanceTracking; bool _isCurrentlyAutoGPUCapturing; + + typedef enum { + VkSemaphoreStyleUseMTLEvent, + VkSemaphoreStyleUseMTLFence, + VkSemaphoreStyleUseEmulation + } VkSemaphoreStyle; + VkSemaphoreStyle _vkSemaphoreStyle; + }; diff --git a/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm b/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm index ebe0114b..b41da9fa 100644 --- a/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm +++ b/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm @@ -3263,12 +3263,10 @@ MVKSemaphore* MVKDevice::createSemaphore(const VkSemaphoreCreateInfo* pCreateInf return new MVKTimelineSemaphoreEmulated(this, pCreateInfo, pTypeCreateInfo); } } else { - if (_useMTLFenceForSemaphores) { - return new MVKSemaphoreMTLFence(this, pCreateInfo); - } else if (_useMTLEventForSemaphores) { - return new MVKSemaphoreMTLEvent(this, pCreateInfo); - } else { - return new MVKSemaphoreEmulated(this, pCreateInfo); + switch (_vkSemaphoreStyle) { + case VkSemaphoreStyleUseMTLEvent: return new MVKSemaphoreMTLEvent(this, pCreateInfo); + case VkSemaphoreStyleUseMTLFence: return new MVKSemaphoreMTLFence(this, pCreateInfo); + case VkSemaphoreStyleUseEmulation: return new MVKSemaphoreEmulated(this, pCreateInfo); } } } @@ -3953,11 +3951,33 @@ void MVKDevice::initPhysicalDevice(MVKPhysicalDevice* physicalDevice, const VkDe _pProperties = &_physicalDevice->_properties; _pMemoryProperties = &_physicalDevice->_memoryProperties; - // Indicate whether semaphores should use a MTLFence or MTLEvent if they are available. - _useMTLFenceForSemaphores = _pMetalFeatures->fences && mvkConfig().semaphoreUseMTLFence; - _useMTLEventForSemaphores = _pMetalFeatures->events && mvkConfig().semaphoreUseMTLEvent; - - MVKLogInfo("Using %s for Vulkan semaphores.", _useMTLFenceForSemaphores ? "MTLFence" : (_useMTLEventForSemaphores ? "MTLEvent" : "emulation")); + // Decide whether semaphores should use a MTLFence or MTLEvent if they are available. + // Prefer MTLEvent over MTLFence, because MTLEvent handles sync across MTLCommandBuffers and MTLCommandQueues, + // except on NVIDIA GPUs, which have demonstrated trouble with MTLEvents, prefer MTLFence. + bool canUseMTLEventForSem4 = _pMetalFeatures->events && mvkConfig().semaphoreUseMTLEvent; + bool canUseMTLFenceForSem4 = _pMetalFeatures->fences && mvkConfig().semaphoreUseMTLFence; + switch (_pProperties->vendorID) { + case kNVVendorId: + _vkSemaphoreStyle = canUseMTLFenceForSem4 ? VkSemaphoreStyleUseMTLFence : (canUseMTLEventForSem4 ? VkSemaphoreStyleUseMTLEvent : VkSemaphoreStyleUseEmulation); + break; + case kAppleVendorId: + case kIntelVendorId: + case kAMDVendorId: + default: + _vkSemaphoreStyle = canUseMTLEventForSem4 ? VkSemaphoreStyleUseMTLEvent : (canUseMTLFenceForSem4 ? VkSemaphoreStyleUseMTLFence : VkSemaphoreStyleUseEmulation); + break; + } + switch (_vkSemaphoreStyle) { + case VkSemaphoreStyleUseMTLEvent: + MVKLogInfo("Using MTLEvent for Vulkan semaphores."); + break; + case VkSemaphoreStyleUseMTLFence: + MVKLogInfo("Using MTLFence for Vulkan semaphores."); + break; + case VkSemaphoreStyleUseEmulation: + MVKLogInfo("Using emulation for Vulkan semaphores."); + break; + } } void MVKDevice::enableFeatures(const VkDeviceCreateInfo* pCreateInfo) { From 8c7db31cd72937a273e070521060ffee9b6a880b Mon Sep 17 00:00:00 2001 From: Bill Hollings Date: Tue, 10 Aug 2021 11:32:21 -0400 Subject: [PATCH 2/2] Prefer MTLEvent for VkSemaphore, except on NVIDIA, prefer emulation. Disable MVK_ALLOW_METAL_FENCES by default. Disable use of MTLEvent on NVIDIA. By default, use MTLEvent for VkSemaphore everywhere except NVIDIA. By default, use CPU synchronization on NVIDIA. These changes fix a large number of CTS synchronization test failures. --- Docs/Whats_New.md | 4 +++- MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h | 17 ++++++++--------- MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm | 21 ++++++--------------- MoltenVK/MoltenVK/Utility/MVKEnvironment.h | 8 ++++---- 4 files changed, 21 insertions(+), 29 deletions(-) diff --git a/Docs/Whats_New.md b/Docs/Whats_New.md index b16a39b7..904ff96c 100644 --- a/Docs/Whats_New.md +++ b/Docs/Whats_New.md @@ -18,7 +18,9 @@ MoltenVK 1.1.5 Released TBD -- Revert to prefer `MTLEvent` over `MTLFence` for `VkSemaphore`, except on NVIDIA. +- Changes to how `VkSemaphore` is supported: + - Revert to prefer `MTLEvent` for `VkSemaphore`, except on NVIDIA, where emulation on CPU is preferred. + - Set default value of the `MVK_ALLOW_METAL_FENCES` environment variable to `0 (false)`, - Vulkan timestamp query pools use Metal GPU counters when available. - Support resolving attachments with formats that Metal does not natively resolve. - Fix issue where swapchain images were acquired out of order under heavy load. diff --git a/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h b/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h index 99138dfe..d3c8a743 100644 --- a/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h +++ b/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h @@ -559,9 +559,9 @@ typedef struct { * * This parameter interacts with semaphoreUseMTLEvent. If both are enabled, on GPUs other than * NVIDIA, semaphoreUseMTLEvent takes priority and MTLEvent will be used if it is available, - * otherwise MTLFence will be used if it is available. On NVIDIA GPUs, the opposite priority - * applies. If neither semaphoreUseMTLFence nor semaphoreUseMTLEvent are enabled, or if neither - * MTLEvent nor MTLFence are available, CPU-based synchoronization will be used. + * otherwise MTLFence will be used if it is available. On NVIDIA GPUs, MTLEvent is disabled + * for VkSemaphores, so CPU-based synchronization will be used unless semaphoreUseMTLFence + * is enabled and MTLFence is available. * * In the special case of VK_SEMAPHORE_TYPE_TIMELINE semaphores, MoltenVK will always * use MTLSharedEvent if it is available on the platform, regardless of the values of @@ -573,8 +573,7 @@ typedef struct { * The initial value or this parameter is set by the * MVK_ALLOW_METAL_FENCES * runtime environment variable or MoltenVK compile-time build setting. - * If neither is set, this setting is enabled by default, and VkSemaphore will use MTLFence, - * if it is available, unless MTLEvent is available and semaphoreUseMTLEvent is enabled. + * If neither is set, this setting is disabled by default, and VkSemaphore will not use MTLFence. */ VkBool32 semaphoreUseMTLFence; @@ -583,9 +582,9 @@ typedef struct { * * This parameter interacts with semaphoreUseMTLFence. If both are enabled, on GPUs other than * NVIDIA, semaphoreUseMTLEvent takes priority and MTLEvent will be used if it is available, - * otherwise MTLFence will be used if it is available. On NVIDIA GPUs, the opposite priority - * applies. If neither semaphoreUseMTLFence nor semaphoreUseMTLEvent are enabled, or if neither - * MTLEvent nor MTLFence are available, CPU-based synchoronization will be used. + * otherwise MTLFence will be used if it is available. On NVIDIA GPUs, MTLEvent is disabled + * for VkSemaphores, so CPU-based synchronization will be used unless semaphoreUseMTLFence + * is enabled and MTLFence is available. * * In the special case of VK_SEMAPHORE_TYPE_TIMELINE semaphores, MoltenVK will always * use MTLSharedEvent if it is available on the platform, regardless of the values of @@ -598,7 +597,7 @@ typedef struct { * MVK_ALLOW_METAL_EVENTS * runtime environment variable or MoltenVK compile-time build setting. * If neither is set, this setting is enabled by default, and VkSemaphore will use MTLEvent, - * if it is available. + * if it is available, except on NVIDIA GPUs. */ VkBool32 semaphoreUseMTLEvent; diff --git a/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm b/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm index b41da9fa..6fd35daa 100644 --- a/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm +++ b/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm @@ -3951,22 +3951,13 @@ void MVKDevice::initPhysicalDevice(MVKPhysicalDevice* physicalDevice, const VkDe _pProperties = &_physicalDevice->_properties; _pMemoryProperties = &_physicalDevice->_memoryProperties; - // Decide whether semaphores should use a MTLFence or MTLEvent if they are available. - // Prefer MTLEvent over MTLFence, because MTLEvent handles sync across MTLCommandBuffers and MTLCommandQueues, - // except on NVIDIA GPUs, which have demonstrated trouble with MTLEvents, prefer MTLFence. - bool canUseMTLEventForSem4 = _pMetalFeatures->events && mvkConfig().semaphoreUseMTLEvent; + // Decide whether Vulkan semaphores should use a MTLEvent or MTLFence if they are available. + // Prefer MTLEvent, because MTLEvent handles sync across MTLCommandBuffers and MTLCommandQueues. + // However, do not allow use of MTLEvents on NVIDIA GPUs, which have demonstrated trouble with MTLEvents. + // Since MTLFence config is disabled by default, emulation will be used on NVIDIA unless MTLFence is enabled. + bool canUseMTLEventForSem4 = _pMetalFeatures->events && mvkConfig().semaphoreUseMTLEvent && (_pProperties->vendorID != kNVVendorId); bool canUseMTLFenceForSem4 = _pMetalFeatures->fences && mvkConfig().semaphoreUseMTLFence; - switch (_pProperties->vendorID) { - case kNVVendorId: - _vkSemaphoreStyle = canUseMTLFenceForSem4 ? VkSemaphoreStyleUseMTLFence : (canUseMTLEventForSem4 ? VkSemaphoreStyleUseMTLEvent : VkSemaphoreStyleUseEmulation); - break; - case kAppleVendorId: - case kIntelVendorId: - case kAMDVendorId: - default: - _vkSemaphoreStyle = canUseMTLEventForSem4 ? VkSemaphoreStyleUseMTLEvent : (canUseMTLFenceForSem4 ? VkSemaphoreStyleUseMTLFence : VkSemaphoreStyleUseEmulation); - break; - } + _vkSemaphoreStyle = canUseMTLEventForSem4 ? VkSemaphoreStyleUseMTLEvent : (canUseMTLFenceForSem4 ? VkSemaphoreStyleUseMTLFence : VkSemaphoreStyleUseEmulation); switch (_vkSemaphoreStyle) { case VkSemaphoreStyleUseMTLEvent: MVKLogInfo("Using MTLEvent for Vulkan semaphores."); diff --git a/MoltenVK/MoltenVK/Utility/MVKEnvironment.h b/MoltenVK/MoltenVK/Utility/MVKEnvironment.h index 4cb724a1..e7d65446 100644 --- a/MoltenVK/MoltenVK/Utility/MVKEnvironment.h +++ b/MoltenVK/MoltenVK/Utility/MVKEnvironment.h @@ -233,15 +233,15 @@ void mvkSetConfig(const MVKConfiguration& mvkConfig); /** * Allow the use of MTLFence or MTLEvent for VkSemaphore synchronization behaviour. * By default: - * - MVK_ALLOW_METAL_FENCES is enabled * - MVK_ALLOW_METAL_EVENTS is enabled + * - MVK_ALLOW_METAL_FENCES is disabled * */ -#ifndef MVK_ALLOW_METAL_FENCES -# define MVK_ALLOW_METAL_FENCES 1 -#endif #ifndef MVK_ALLOW_METAL_EVENTS # define MVK_ALLOW_METAL_EVENTS 1 #endif +#ifndef MVK_ALLOW_METAL_FENCES +# define MVK_ALLOW_METAL_FENCES 0 +#endif /** Substitute Metal 2D textures for Vulkan 1D images. Enabled by default. */ #ifndef MVK_CONFIG_TEXTURE_1D_AS_2D