From 35b465d6c62084ac1b3e547aaeee56b9c1978358 Mon Sep 17 00:00:00 2001
From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com>
Date: Fri, 9 Aug 2024 14:07:50 +0200
Subject: [PATCH] Recast: don't link poly edges facing the same direction

Only link them if they are perpendicular or facing the opposite direction.
---
 .../Detour/Source/DetourNavMeshBuilder.cpp     | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/thirdparty/recast/Detour/Source/DetourNavMeshBuilder.cpp b/src/thirdparty/recast/Detour/Source/DetourNavMeshBuilder.cpp
index a49009f6..c40c9fb3 100644
--- a/src/thirdparty/recast/Detour/Source/DetourNavMeshBuilder.cpp
+++ b/src/thirdparty/recast/Detour/Source/DetourNavMeshBuilder.cpp
@@ -487,6 +487,8 @@ static void connectTileTraverseLinks(dtNavMesh* const nav, dtMeshTile* const til
 			startPolyEdgeMid[1] = (startPolySpos[1]+startPolyEpos[1])*0.5f;
 			startPolyEdgeMid[2] = (startPolySpos[2]+startPolyEpos[2])*0.5f;
 
+			float startEdgeDir[3];
+			rdVsub(startEdgeDir, startPolyEpos, startPolySpos);
 
 			for (int k = 0; k < tile->header->polyCount; ++k)
 			{
@@ -510,7 +512,21 @@ static void connectTileTraverseLinks(dtNavMesh* const nav, dtMeshTile* const til
 					endPolyEdgeMid[1] = (endPolySpos[1]+endPolyEpos[1])*0.5f;
 					endPolyEdgeMid[2] = (endPolySpos[2]+endPolyEpos[2])*0.5f;
 
-					// TODO: calculate edge midpoint first !!!
+					float endEdgeDir[3];
+					rdVsub(endEdgeDir, endPolyEpos, endPolySpos);
+
+					const float dotProduct = rdVdot(startEdgeDir, endEdgeDir);
+
+					// Edges facing the same direction should not be linked.
+					// Doing so causes links to go through from underneath
+					// geometry. E.g. we have an HVAC on a roof, and we try
+					// to link our roof poly edge facing north to the edge
+					// of the poly on the HVAC also facing north, the link
+					// will go through the HVAC and thus cause the NPC to
+					// jump through it.
+					if (dotProduct > 0)
+						continue;
+
 					const unsigned char distance = dtCalcLinkDistance(startPolyEdgeMid, endPolyEdgeMid);
 
 					// TODO: needs lookup table for distance !!!