From 77b9d056e8c2b83daf19eafee419644fcfe8aef0 Mon Sep 17 00:00:00 2001
From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com>
Date: Fri, 30 Aug 2024 15:51:28 +0200
Subject: [PATCH] Recast: properly flag polygons after creation

Titanfall 2 navmeshes flag their polygons as follows. The pattern was always that if a polygon connects to a polygon on a neighboring tile, it should be flagged as   EDITOR_POLYFLAGS_HAS_NEIGHBOUR, and if the polygon's surface area isn't higher than 120 (NAVMESH_SMALL_POLYGON_THRESHOLD) it should be flagged as EDITOR_POLYFLAGS_TOO_SMALL.
---
 src/naveditor/Editor_TileMesh.cpp   | 19 +++++++++++++++++++
 src/naveditor/NavMeshTesterTool.cpp |  8 ++++----
 src/naveditor/include/Editor.h      |  8 ++++++--
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/src/naveditor/Editor_TileMesh.cpp b/src/naveditor/Editor_TileMesh.cpp
index e3576fb2..8b1bed74 100644
--- a/src/naveditor/Editor_TileMesh.cpp
+++ b/src/naveditor/Editor_TileMesh.cpp
@@ -1260,6 +1260,25 @@ unsigned char* Editor_TileMesh::buildTileMesh(const int tx, const int ty, const
 			{
 				m_pmesh->flags[i] = EDITOR_POLYFLAGS_WALK /*| EDITOR_POLYFLAGS_DOOR*/;
 			}
+
+			if (m_pmesh->surfa[i] <= NAVMESH_SMALL_POLYGON_THRESHOLD)
+				m_pmesh->flags[i] |= EDITOR_POLYFLAGS_TOO_SMALL;
+
+			const int nvp = m_pmesh->nvp;
+			const unsigned short* p = &m_pmesh->polys[i*nvp*2];
+
+			// If polygon connects to a polygon on a neighbouring tile, flag it.
+			for (int j = 0; j < nvp; ++j)
+			{
+				if (p[j] == RC_MESH_NULL_IDX)
+					break;
+				if ((p[nvp+j] & 0x8000) == 0)
+					continue;
+				if ((p[nvp+j] & 0xf) == 0xf)
+					continue;
+
+				m_pmesh->flags[i] |= EDITOR_POLYFLAGS_HAS_NEIGHBOUR;
+			}
 		}
 		
 		dtNavMeshCreateParams params;
diff --git a/src/naveditor/NavMeshTesterTool.cpp b/src/naveditor/NavMeshTesterTool.cpp
index 07886f9a..b1b9df15 100644
--- a/src/naveditor/NavMeshTesterTool.cpp
+++ b/src/naveditor/NavMeshTesterTool.cpp
@@ -394,11 +394,11 @@ void NavMeshTesterTool::handleMenu()
 		recalc();
 	}
 
-	isEnabled = (m_filter.getIncludeFlags() & EDITOR_POLYFLAGS_SKIP) != 0;
+	isEnabled = (m_filter.getIncludeFlags() & EDITOR_POLYFLAGS_TOO_SMALL) != 0;
 
 	if (ImGui::Checkbox("Skip##IncludeFlags", &isEnabled))
 	{
-		m_filter.setIncludeFlags(m_filter.getIncludeFlags() ^ EDITOR_POLYFLAGS_SKIP);
+		m_filter.setIncludeFlags(m_filter.getIncludeFlags() ^ EDITOR_POLYFLAGS_TOO_SMALL);
 		recalc();
 	}
 
@@ -417,11 +417,11 @@ void NavMeshTesterTool::handleMenu()
 		recalc();
 	}
 
-	isEnabled = (m_filter.getExcludeFlags() & EDITOR_POLYFLAGS_SKIP) != 0;
+	isEnabled = (m_filter.getExcludeFlags() & EDITOR_POLYFLAGS_TOO_SMALL) != 0;
 
 	if (ImGui::Checkbox("Skip##ExcludeFlags", &isEnabled))
 	{
-		m_filter.setExcludeFlags(m_filter.getExcludeFlags() ^ EDITOR_POLYFLAGS_SKIP);
+		m_filter.setExcludeFlags(m_filter.getExcludeFlags() ^ EDITOR_POLYFLAGS_TOO_SMALL);
 		recalc();
 	}
 
diff --git a/src/naveditor/include/Editor.h b/src/naveditor/include/Editor.h
index 0a3f0a91..276b4e55 100644
--- a/src/naveditor/include/Editor.h
+++ b/src/naveditor/include/Editor.h
@@ -169,12 +169,16 @@ enum EditorPolyAreas
 //	EDITOR_POLYFLAGS_ALL		= 0xffff	// All abilities.
 //};
 
+// Polygon surface area's that aren't larger than this amount will be flagged
+// as 'EDITOR_POLYFLAGS_TOO_SMALL'.
+static const unsigned short NAVMESH_SMALL_POLYGON_THRESHOLD = 120;
+
 enum EditorPolyFlags
 {
 	// Most common polygon flags.
 	EDITOR_POLYFLAGS_WALK				= 1<<0,		// Ability to walk (ground, grass, road).
-	EDITOR_POLYFLAGS_SKIP				= 1<<1,     // Skipped during AIN script nodes generation, NavMesh_RandomPositions, dtNavMeshQuery::findLocalNeighbourhood, etc.
-	EDITOR_POLYFLAGS_UNK0				= 1<<2,     // Unknown, most polygon have this flag.
+	EDITOR_POLYFLAGS_TOO_SMALL			= 1<<1,     // This polygon's surface area is too small; it will be ignored during AIN script nodes generation, NavMesh_RandomPositions, dtNavMeshQuery::findLocalNeighbourhood, etc.
+	EDITOR_POLYFLAGS_HAS_NEIGHBOUR		= 1<<2,     // This polygon is connected to a polygon on a neighbouring tile.
 
 	// Off-mesh connection flags
 	EDITOR_POLYFLAGS_JUMP				= 1<<3,		// Ability to jump (exclusively used on off-mesh connection polygons).