Recast: tag semi and fully unlinked tiles with userid's

Fully unlinked can be removed entirely, semi unlinked needs a rebuild to remove unlinked polygons (still work in progress). Added ability to automatically remove fully unlinked tiles.

src/naveditor/NavMeshPruneTool.cpp

@@ -128,7 +128,7 @@ static void floodNavmesh(dtNavMesh* nav, NavmeshFlags* flags, dtPolyRef start, u
 
 	flags->setFlags(start, flag);
 		
-	std::vector<dtPolyRef> openList;
+	rdPermVector<dtPolyRef> openList;
 	openList.push_back(start);
 
 	while (openList.size())
@@ -161,7 +161,7 @@ static void disableUnvisitedPolys(dtNavMesh* nav, NavmeshFlags* flags)
 {
 	for (int i = 0; i < nav->getTileCount(); ++i)
 	{
-		const dtMeshTile* tile = ((const dtNavMesh*)nav)->getTile(i);
+		const dtMeshTile* tile = nav->getTile(i);
 		dtMeshHeader* header = tile->header;
 
 		if (!header) continue;
@@ -178,23 +178,45 @@ static void disableUnvisitedPolys(dtNavMesh* nav, NavmeshFlags* flags)
 				dtPoly* targetPoly;
 
 				nav->getTileAndPolyByRefUnsafe(ref, &targetTile, (const dtPoly**)&targetPoly);
+
 				targetPoly->groupId = DT_UNLINKED_POLY_GROUP;
 				targetPoly->firstLink = DT_NULL_LINK;
-				targetPoly->flags = 0;
+				targetPoly->flags = EDITOR_POLYFLAGS_DISABLED;
 
 				numUnlinkedPolys++;
 			}
 		}
 
 		if (numUnlinkedPolys == header->polyCount)
-			header->userId = DT_UNLINKED_TILE_USER_ID;
+		{
+			header->userId = DT_FULL_UNLINKED_TILE_USER_ID;
+			continue;
+		}
 	}
 }
 
+static void removeUnlinkedTiles(dtNavMesh* nav)
+{
+	for (int i = nav->getTileCount(); i-- > 0;)
+	{
+		const dtMeshTile* tile = nav->getTile(i);
+		const dtMeshHeader* header = tile->header;
+
+		if (!header) continue;
+
+		if (header->userId == DT_FULL_UNLINKED_TILE_USER_ID)
+		{
+			const int polyCount = header->polyCount;
+			nav->removeTile(nav->getTileRef(tile), 0, 0);
+		}
+	};
+}
+
 NavMeshPruneTool::NavMeshPruneTool() :
 	m_editor(0),
 	m_flags(0),
-	m_hitPosSet(false)
+	m_hitPosSet(false),
+	m_ranPruneTool(false)
 {
 	m_hitPos[0] = 0.0f;
 	m_hitPos[1] = 0.0f;
@@ -214,6 +236,7 @@ void NavMeshPruneTool::init(Editor* editor)
 void NavMeshPruneTool::reset()
 {
 	m_hitPosSet = false;
+	m_ranPruneTool = false;
 	delete m_flags;
 	m_flags = 0;
 }
@@ -222,6 +245,14 @@ void NavMeshPruneTool::handleMenu()
 {
 	dtNavMesh* nav = m_editor->getNavMesh();
 	if (!nav) return;
+
+	// todo(amos): once tile rebuilding is done, also remove unlinked polygons!
+	if (m_ranPruneTool && ImGui::Button("Remove Unlinked Tiles"))
+	{
+		removeUnlinkedTiles(nav);
+		m_ranPruneTool = false;
+	}
+
 	if (!m_flags) return;
 
 	if (ImGui::Button("Clear Selection"))
@@ -239,6 +270,8 @@ void NavMeshPruneTool::handleMenu()
 
 		delete m_flags;
 		m_flags = 0;
+
+		m_ranPruneTool = true;
 	}
 }
 

src/naveditor/include/NavMeshPruneTool.h

@@ -31,6 +31,7 @@ class NavMeshPruneTool : public EditorTool
 
 	float m_hitPos[3];
 	bool m_hitPosSet;
+	bool m_ranPruneTool;
 	
 public:
 	NavMeshPruneTool();

src/thirdparty/recast/Detour/Include/DetourNavMesh.h

@@ -66,7 +66,11 @@ typedef unsigned int dtTileRef;
 /// to the rest of the reachable area's of the navigation mesh, this tile will not be
 /// added to the position lookup table.
 /// @ingroup detour
-static const int DT_UNLINKED_TILE_USER_ID = 1;
+static const int DT_FULL_UNLINKED_TILE_USER_ID = 1;
+
+/// A value that indicates that this tile contains at least 1 polygon that doesn't link
+/// to anything (tagged as #DT_UNLINKED_POLY_GROUP), and 1 that does link to something.
+static const int DT_SEMI_UNLINKED_TILE_USER_ID = 2;
 
 /// The maximum number of vertices per navigation polygon.
 /// @ingroup detour
@@ -899,9 +903,8 @@ private:
 	///< See note at dtNavMeshParams::magicDataCount for buffer allocation.
 	void* m_someMagicData;
 
-	char m_meshFlags;	// Maybe.
-	char m_tileFlags;	// Maybe.
-	int m_unk1;			// FIXME:
+	int m_unused0;
+	int m_unused1;
 
 	dtNavMeshParams m_params;			///< Current initialization params. TODO: do not store this info twice.
 	float m_orig[3];					///< Origin of the tile (0,0)

src/thirdparty/recast/Detour/Source/DetourNavMesh.cpp

@@ -256,9 +256,8 @@ dtNavMesh::dtNavMesh() :
 	m_tiles(0),
 	m_traverseTables(0),
 	m_someMagicData(0),
-	m_meshFlags(0),
-	m_tileFlags(0),
-	m_unk1(0)
+	m_unused0(0),
+	m_unused1(0)
 {
 #ifndef DT_POLYREF64
 	m_saltBits = 0;
@@ -1045,7 +1044,7 @@ dtStatus dtNavMesh::addTile(unsigned char* data, int dataSize, int flags,
 #endif
 		
 	// Make sure the location is free.
-	if (!header->userId != DT_UNLINKED_TILE_USER_ID && getTileAt(header->x, header->y, header->layer))
+	if (!header->userId != DT_FULL_UNLINKED_TILE_USER_ID && getTileAt(header->x, header->y, header->layer))
 		return DT_FAILURE | DT_ALREADY_OCCUPIED;
 		
 	// Allocate a tile.
@@ -1094,7 +1093,7 @@ dtStatus dtNavMesh::addTile(unsigned char* data, int dataSize, int flags,
 	tile->deleteCallback = nullptr;
 	
 	// Insert tile into the position lut.
-	if (header->userId != DT_UNLINKED_TILE_USER_ID)
+	if (header->userId != DT_FULL_UNLINKED_TILE_USER_ID)
 	{
 		int h = computeTileHash(header->x, header->y, m_tileLutMask);
 		tile->next = m_posLookup[h];

src/thirdparty/recast/Detour/Source/DetourNavMeshBuilder.cpp

@@ -416,6 +416,7 @@ bool dtUpdateDisjointPolyGroups(const dtTraverseTableCreateParams* params)
 		dtMeshTile* tile = nav->getTile(i);
 		if (!tile || !tile->header || !tile->dataSize) continue;
 		const int pcount = tile->header->polyCount;
+		int numUnlinkedPolys = 0;
 		for (int j = 0; j < pcount; j++)
 		{
 			dtPoly& poly = tile->polys[j];
@@ -423,8 +424,21 @@ bool dtUpdateDisjointPolyGroups(const dtTraverseTableCreateParams* params)
 			// This poly isn't connected to anything, mark it so the game
 			// won't consider this poly in path generation.
 			if (poly.firstLink == DT_NULL_LINK)
+			{
 				poly.groupId = DT_UNLINKED_POLY_GROUP;
+				numUnlinkedPolys++;
+			}
 		}
+
+		if (numUnlinkedPolys)
+		{
+			tile->header->userId = (numUnlinkedPolys == tile->header->polyCount)
+				? DT_FULL_UNLINKED_TILE_USER_ID
+				: DT_SEMI_UNLINKED_TILE_USER_ID;
+		}
+		else if (tile->header->userId == DT_FULL_UNLINKED_TILE_USER_ID 
+			|| tile->header->userId == DT_SEMI_UNLINKED_TILE_USER_ID)
+			tile->header->userId = 0;
 	}
 
 	// Gather all unique polygroups and map them to a contiguous range.