Recast: traverse linking code cleanup and improvements

Split traverse link creation and updating code from the editor, only alloc traverse table when we are creating them or when we load navmeshes that have them, don't alloc them before the navmesh is built. Also moved the disjoint set data to the editor class so we can cache the results and reuse them later after editing the navmesh post build. Traverse link algorithm now also ignores polys marked unlinked.

src/naveditor/Editor.cpp

@@ -659,6 +659,9 @@ void Editor::connectTileTraverseLinks(dtMeshTile* const baseTile, const bool lin
 	{
 		dtPoly* const basePoly = &baseTile->polys[i];
 
+		if (basePoly->groupId == DT_STRAY_POLY_GROUP)
+			continue;
+
 		for (int j = 0; j < basePoly->vertCount; ++j)
 		{
 			// Hard edges only!
@@ -698,6 +701,9 @@ void Editor::connectTileTraverseLinks(dtMeshTile* const baseTile, const bool lin
 				{
 					dtPoly* const landPoly = &landTile->polys[m];
 
+					if (landPoly->groupId == DT_STRAY_POLY_GROUP)
+						continue;
+
 					for (int n = 0; n < landPoly->vertCount; ++n)
 					{
 						if (landPoly->neis[n] != 0)
@@ -831,31 +837,48 @@ bool Editor::createTraverseLinks()
 	return true;
 }
 
-void Editor::buildStaticPathingData()
+bool Editor::createStaticPathingData()
 {
-	if (!m_navMesh) return;
+	if (!m_navMesh) return false;
 
-	dtDisjointSet data;
-
-	if (!dtCreateDisjointPolyGroups(m_navMesh, data))
+	if (!dtCreateDisjointPolyGroups(m_navMesh, m_djs))
 	{
-		m_ctx->log(RC_LOG_ERROR, "buildStaticPathingData: Failed to build disjoint poly groups.");
+		m_ctx->log(RC_LOG_ERROR, "createStaticPathingData: Failed to build disjoint poly groups.");
+		return false;
 	}
 
 	if (!createTraverseLinks())
 	{
-		m_ctx->log(RC_LOG_ERROR, "buildStaticPathingData: Failed to build traverse links.");
+		m_ctx->log(RC_LOG_ERROR, "createStaticPathingData: Failed to build traverse links.");
+		return false;
 	}
 
-	if (!dtUpdateDisjointPolyGroups(m_navMesh, data))
+	return true;
+}
+
+bool Editor::updateStaticPathingData()
+{
+	if (!m_navMesh) return false;
+
+	if (!dtUpdateDisjointPolyGroups(m_navMesh, m_djs))
 	{
-		m_ctx->log(RC_LOG_ERROR, "buildStaticPathingData: Failed to update disjoint poly groups.");
+		m_ctx->log(RC_LOG_ERROR, "updateStaticPathingData: Failed to update disjoint poly groups.");
+		return false;
 	}
 
-	if (!dtCreateTraverseTableData(m_navMesh, data, NavMesh_GetTraverseTableCountForNavMeshType(m_selectedNavMeshType)))
+	if (!dtCreateTraverseTableData(m_navMesh, m_djs, NavMesh_GetTraverseTableCountForNavMeshType(m_selectedNavMeshType)))
 	{
-		m_ctx->log(RC_LOG_ERROR, "buildStaticPathingData: Failed to build traverse table data.");
+		m_ctx->log(RC_LOG_ERROR, "updateStaticPathingData: Failed to build traverse table data.");
+		return false;
 	}
+
+	return true;
+}
+
+void Editor::buildStaticPathingData()
+{
+	createStaticPathingData();
+	updateStaticPathingData();
 }
 
 void Editor::updateToolStates(const float dt)

src/naveditor/Editor_TileMesh.cpp

@@ -484,7 +484,7 @@ bool Editor_TileMesh::handleBuild()
 	params.maxPolys = m_maxPolysPerTile;
 	params.polyGroupCount = 0;
 	params.traverseTableSize = 0;
-	params.traverseTableCount = NavMesh_GetTraverseTableCountForNavMeshType(m_selectedNavMeshType);
+	params.traverseTableCount = 0;
 #if DT_NAVMESH_SET_VERSION >= 8
 	params.magicDataCount = 0;
 #endif

src/naveditor/include/Editor.h

@@ -24,6 +24,8 @@
 #include "DebugUtils/Include/RecastDebugDraw.h"
 #include "DebugUtils/Include/DetourDebugDraw.h"
 
+#include "Detour/Include/DetourNavMeshBuilder.h"
+
 #include "game/server/ai_navmesh.h"
 
 struct dtMeshTile;
@@ -171,6 +173,7 @@ protected:
 	EditorToolState* m_toolStates[MAX_TOOLS];
 	
 	BuildContext* m_ctx;
+	dtDisjointSet m_djs;
 
 	EditorDebugDraw m_dd;
 	unsigned int m_navMeshDrawFlags;
@@ -249,6 +252,9 @@ public:
 	bool createTraverseLinks();
 	void buildStaticPathingData();
 
+	bool createStaticPathingData();
+	bool updateStaticPathingData();
+
 private:
 	// Explicitly disabled copy constructor and copy assignment operator.
 	Editor(const Editor&);

src/thirdparty/recast/Detour/Include/DetourNavMesh.h

@@ -644,6 +644,9 @@ public:
 	/// @return The specified off-mesh connection, or null if the polygon reference is not valid.
 	const dtOffMeshConnection* getOffMeshConnectionByRef(dtPolyRef ref) const;
 
+	bool allocTraverseTables(const int count);
+	void freeTraverseTables();
+
 	/// The navigation mesh traverse tables.
 	int** getTraverseTables() const { return m_traverseTables; }
 	
@@ -652,6 +655,10 @@ public:
 	///  @param[in]	table	The traverse table data.
 	void setTraverseTable(const int index, int* const table);
 
+	/// Sets the number of the traverse tables.
+	///  @param[in]	count	The number of the traverse tables.
+	void setTraverseTableCount(const int count) { m_params.traverseTableCount = count; }
+
 	/// Sets the size of the traverse table.
 	///  @param[in]	size	The size of the traverse table.
 	void setTraverseTableSize(const int size) { m_params.traverseTableSize = size; }

src/thirdparty/recast/Detour/Source/DetourNavMesh.cpp

@@ -231,15 +231,7 @@ dtNavMesh::~dtNavMesh() // TODO: see [r5apex_ds + F43720] to re-implement this c
 	rdFree(m_posLookup);
 	rdFree(m_tiles);
 
-	for (int i = 0; i < m_params.traverseTableCount; i++)
-	{
-		int* traverseTable = m_traverseTables[i];
-
-		if (traverseTable)
-			rdFree(traverseTable);
-	}
-
-	rdFree(m_traverseTables);
+	freeTraverseTables();
 }
 		
 dtStatus dtNavMesh::init(const dtNavMeshParams* params)
@@ -268,14 +260,8 @@ dtStatus dtNavMesh::init(const dtNavMeshParams* params)
 	const int traverseTableCount = params->traverseTableCount;
 	if (traverseTableCount)
 	{
-		rdAssert(traverseTableCount > 0 && traverseTableCount <= DT_MAX_TRAVERSE_TABLES);
-		const int setTableBufSize = sizeof(int**)*traverseTableCount;
-
-		m_traverseTables = (int**)rdAlloc(setTableBufSize, RD_ALLOC_PERM);
-		if (!m_traverseTables)
+		if (!allocTraverseTables(params->traverseTableCount))
 			return DT_FAILURE | DT_OUT_OF_MEMORY;
-
-		memset(m_traverseTables, 0, setTableBufSize);
 	}
 
 	m_nextFree = 0;
@@ -1655,12 +1641,40 @@ const dtOffMeshConnection* dtNavMesh::getOffMeshConnectionByRef(dtPolyRef ref) c
 	return &tile->offMeshCons[idx];
 }
 
+bool dtNavMesh::allocTraverseTables(const int count)
+{
+	rdAssert(count > 0 && count <= DT_MAX_TRAVERSE_TABLES);
+	const int setTableBufSize = sizeof(int**) * count;
+
+	m_traverseTables = (int**)rdAlloc(setTableBufSize, RD_ALLOC_PERM);
+	if (!m_traverseTables)
+		return false;
+
+	memset(m_traverseTables, 0, setTableBufSize);
+	return true;
+}
+
+void dtNavMesh::freeTraverseTables()
+{
+	for (int i = 0; i < m_params.traverseTableCount; i++)
+	{
+		int* traverseTable = m_traverseTables[i];
+
+		if (traverseTable)
+			rdFree(traverseTable);
+	}
+
+	rdFree(m_traverseTables);
+}
 
 void dtNavMesh::setTraverseTable(const int index, int* const table)
 {
 	rdAssert(index >= 0 && index < m_params.traverseTableCount);
 	rdAssert(m_traverseTables);
 
+	if (m_traverseTables[index])
+		rdFree(m_traverseTables[index]);
+
 	m_traverseTables[index] = table;
 }
 

src/thirdparty/recast/Detour/Source/DetourNavMeshBuilder.cpp

@@ -266,7 +266,9 @@ bool dtCreateDisjointPolyGroups(dtNavMesh* nav, dtDisjointSet& disjoint)
 		for (int j = 0; j < pcount; j++)
 		{
 			dtPoly& poly = tile->polys[j];
-			poly.groupId = DT_NULL_POLY_GROUP;
+
+			if (poly.groupId != DT_STRAY_POLY_GROUP)
+				poly.groupId = DT_NULL_POLY_GROUP;
 
 #if DT_NAVMESH_SET_VERSION >= 7
 			// NOTE: these fields are unknown and need to be reversed.
@@ -341,7 +343,7 @@ bool dtCreateDisjointPolyGroups(dtNavMesh* nav, dtDisjointSet& disjoint)
 			dtPoly& poly = tile->polys[j];
 			if (poly.groupId != DT_STRAY_POLY_GROUP)
 			{
-				int id = disjoint.find(poly.groupId);
+				const int id = disjoint.find(poly.groupId);
 				poly.groupId = (unsigned short)id;
 			}
 		}
@@ -500,23 +502,16 @@ bool dtCreateTraverseTableData(dtNavMesh* nav, const dtDisjointSet& disjoint, co
 	const int polyGroupCount = nav->getPolyGroupCount();
 	const int tableSize = dtCalcTraverseTableSize(polyGroupCount);
 
-	// TODO: currently we allocate 5 buffers and just copy the same traverse
-	// tables in, this works fine since we don't generate jump links and
-	// therefore all poly islands should be marked unreachable from each other.
-	// But when we generate jump links, we need to take into consideration that
-	// the '_small' navmesh supports 5 animation types (dictated by the field
-	// "TraverseAnimType" in the NPC's settings file, and each of them have
-	// different properties in regards to how far they can jump, or the angle,
-	// ect... For example the "frag_drone" anim type can take far further jumps
-	// than the "human" one. The human indexes into the first table while 
-	// frag_drone indexes into the fourth. We have to set different links per
-	// table. The 'dtLink::jumpType' field probably determines what belongs to
-	// what TraverseAnimType, which we could use to set the traversability. 
-	// More reasearch is needed for the jump links and flags... For other
-	// navmeshes, e.g. the '_large' one, they all contain only 1 traverse
-	// table as they only support one TraverseAnimType each. but also here
-	// we have to "reverse" the properties from existing Titanfall 2 single
-	// player navmeshes and determine the traversability in this loop below.
+	nav->freeTraverseTables();
+
+	if (!nav->allocTraverseTables(tableCount))
+		return false;
+
+	nav->setTraverseTableSize(tableSize);
+	nav->setTraverseTableCount(tableCount);
+
+	// TODO: figure out which jump type belongs to which traverse anim type
+	// and determine reachability from here...
 	for (int i = 0; i < tableCount; i++)
 	{
 		int* const traverseTable = (int*)rdAlloc(sizeof(int)*tableSize, RD_ALLOC_PERM);
@@ -538,8 +533,6 @@ bool dtCreateTraverseTableData(dtNavMesh* nav, const dtDisjointSet& disjoint, co
 		}
 	}
 
-	nav->setTraverseTableSize(tableSize);
-
 	return true;
 }