Recast: add logic for setting polygon flags in convex hulls

User can now define a convex hull or box area, and set polygons flags as desired from the convex volume tool or through the project files.

src/naveditor/ConvexVolumeTool.cpp

@@ -92,6 +92,7 @@ static int pointInPoly(int nvert, const float* verts, const float* p) // todo(am
 ConvexVolumeTool::ConvexVolumeTool() :
 	m_editor(0),
 	m_areaType(RC_NULL_AREA),
+	m_polyFlags(0),
 	m_polyOffset(0.0f),
 	m_boxHeight(650.0f),
 	m_boxDescent(150.0f),
@@ -135,8 +136,21 @@ void ConvexVolumeTool::handleMenu()
 	if (ImGui::Checkbox("Trigger", &isEnabled))
 		m_areaType = EDITOR_POLYAREA_TRIGGER; // todo(amos): also allow setting flags and store this in .gset.
 
-	ImGui::Unindent();
+	if (m_areaType == EDITOR_POLYAREA_TRIGGER)
+	{
+		ImGui::Text("Poly Flags");
+		ImGui::Indent();
 
+		for (int i = 0; i < V_ARRAYSIZE(g_navMeshPolyFlagNames); i++)
+		{
+			const char* flagName = g_navMeshPolyFlagNames[i];
+			ImGui::CheckboxFlags(flagName, &m_polyFlags, 1<<i);
+		}
+
+		ImGui::Unindent();
+	}
+
+	ImGui::Unindent();
 	ImGui::Separator();
 
 	if (ImGui::Button("Clear Shape"))
@@ -196,11 +210,11 @@ void ConvexVolumeTool::handleClick(const float* /*s*/, const float* p, bool shif
 					float offset[MAX_PTS*2*3];
 					int noffset = rcOffsetPoly(verts, m_nhull, m_polyOffset, offset, MAX_PTS*2);
 					if (noffset > 0)
-						geom->addConvexVolume(offset, noffset, minh, maxh, (unsigned char)m_areaType);
+						geom->addConvexVolume(offset, noffset, minh, maxh, (unsigned short)m_polyFlags, (unsigned char)m_areaType);
 				}
 				else
 				{
-					geom->addConvexVolume(verts, m_nhull, minh, maxh, (unsigned char)m_areaType);
+					geom->addConvexVolume(verts, m_nhull, minh, maxh, (unsigned short)m_polyFlags, (unsigned char)m_areaType);
 				}
 			}
 			

src/naveditor/Editor_SoloMesh.cpp

@@ -301,7 +301,7 @@ bool Editor_SoloMesh::handleBuild()
 	// (Optional) Mark areas.
 	const ConvexVolume* vols = m_geom->getConvexVolumes();
 	for (int i  = 0; i < m_geom->getConvexVolumeCount(); ++i)
-		rcMarkConvexPolyArea(m_ctx, vols[i].verts, vols[i].nverts, vols[i].hmin, vols[i].hmax, (unsigned char)vols[i].area, *m_chf);
+		rcMarkConvexPolyArea(m_ctx, vols[i].verts, vols[i].nverts, vols[i].hmin, vols[i].hmax, (unsigned short)vols[i].flags, (unsigned char)vols[i].area, *m_chf);
 
 	
 	// Partition the heightfield so that we can use simple algorithm later to triangulate the walkable areas.

src/naveditor/Editor_TempObstacles.cpp

@@ -402,6 +402,7 @@ int Editor_TempObstacles::rasterizeTileLayers(
 	{
 		rcMarkConvexPolyArea(m_ctx, vols[i].verts, vols[i].nverts,
 							 vols[i].hmin, vols[i].hmax,
+							 (unsigned short)vols[i].flags,
 							 (unsigned char)vols[i].area, *rc.chf);
 	}
 	

src/naveditor/Editor_TileMesh.cpp

@@ -1108,7 +1108,7 @@ unsigned char* Editor_TileMesh::buildTileMesh(const int tx, const int ty, const
 	// (Optional) Mark areas.
 	const ConvexVolume* vols = m_geom->getConvexVolumes();
 	for (int i  = 0; i < m_geom->getConvexVolumeCount(); ++i)
-		rcMarkConvexPolyArea(m_ctx, vols[i].verts, vols[i].nverts, vols[i].hmin, vols[i].hmax, (unsigned char)vols[i].area, *m_chf);
+		rcMarkConvexPolyArea(m_ctx, vols[i].verts, vols[i].nverts, vols[i].hmin, vols[i].hmax, (unsigned short)vols[i].flags, (unsigned char)vols[i].area, *m_chf);
 	
 	
 	// Partition the heightfield so that we can use simple algorithm later to triangulate the walkable areas.
@@ -1253,7 +1253,7 @@ unsigned char* Editor_TileMesh::buildTileMesh(const int tx, const int ty, const
 				//m_pmesh->areas[i] == EDITOR_POLYAREA_ROAD
 				)
 			{
-				m_pmesh->flags[i] = EDITOR_POLYFLAGS_WALK;
+				m_pmesh->flags[i] |= EDITOR_POLYFLAGS_WALK;
 			}
 			//else if (m_pmesh->areas[i] == EDITOR_POLYAREA_WATER)
 			//{
@@ -1261,7 +1261,7 @@ unsigned char* Editor_TileMesh::buildTileMesh(const int tx, const int ty, const
 			//}
 			else if (m_pmesh->areas[i] == EDITOR_POLYAREA_TRIGGER)
 			{
-				m_pmesh->flags[i] = EDITOR_POLYFLAGS_WALK /*| EDITOR_POLYFLAGS_DOOR*/;
+				m_pmesh->flags[i] |= EDITOR_POLYFLAGS_WALK /*| EDITOR_POLYFLAGS_DOOR*/;
 			}
 
 			if (m_pmesh->surfa[i] <= NAVMESH_SMALL_POLYGON_THRESHOLD)

src/naveditor/InputGeom.cpp

@@ -312,7 +312,7 @@ bool InputGeom::loadGeomSet(rcContext* ctx, const std::string& filepath)
 			if (m_volumeCount < MAX_VOLUMES)
 			{
 				ConvexVolume* vol = &m_volumes[m_volumeCount++];
-				sscanf(row+1, "%d %d %f %f", &vol->nverts, &vol->area, &vol->hmin, &vol->hmax);
+				sscanf(row+1, "%d %hu %hhu %f %f", &vol->nverts, &vol->flags, &vol->area, &vol->hmin, &vol->hmax);
 				for (int i = 0; i < vol->nverts; ++i)
 				{
 					row[0] = '\0';
@@ -453,7 +453,7 @@ bool InputGeom::saveGeomSet(const BuildSettings* settings)
 	for (int i = 0; i < m_volumeCount; ++i)
 	{
 		ConvexVolume* vol = &m_volumes[i];
-		fprintf(fp, "v %d %d %f %f\n", vol->nverts, vol->area, vol->hmin, vol->hmax);
+		fprintf(fp, "v %d %hu %hhu %f %f\n", vol->nverts, vol->flags, vol->area, vol->hmin, vol->hmax);
 		for (int j = 0; j < vol->nverts; ++j)
 			fprintf(fp, "%f %f %f\n", vol->verts[j*3+0], vol->verts[j*3+1], vol->verts[j*3+2]);
 	}
@@ -637,7 +637,7 @@ void InputGeom::drawOffMeshConnections(duDebugDraw* dd, const float* offset, boo
 }
 
 void InputGeom::addConvexVolume(const float* verts, const int nverts,
-								const float minh, const float maxh, unsigned char area)
+								const float minh, const float maxh, unsigned short flags, unsigned char area)
 {
 	if (m_volumeCount >= MAX_VOLUMES) return;
 	ConvexVolume* vol = &m_volumes[m_volumeCount++];
@@ -646,6 +646,7 @@ void InputGeom::addConvexVolume(const float* verts, const int nverts,
 	vol->hmin = minh;
 	vol->hmax = maxh;
 	vol->nverts = nverts;
+	vol->flags = flags;
 	vol->area = area;
 }
 

src/naveditor/include/ConvexVolumeTool.h

@@ -27,6 +27,7 @@ class ConvexVolumeTool : public EditorTool
 {
 	Editor* m_editor;
 	int m_areaType;
+	int m_polyFlags;
 	float m_polyOffset;
 	float m_boxHeight;
 	float m_boxDescent;

src/naveditor/include/InputGeom.h

@@ -28,7 +28,8 @@ struct ConvexVolume
 	float verts[MAX_CONVEXVOL_PTS*3];
 	float hmin, hmax;
 	int nverts;
-	int area;
+	unsigned short flags;
+	unsigned char area;
 };
 
 struct BuildSettings
@@ -160,7 +161,7 @@ public:
 	int getConvexVolumeCount() const { return m_volumeCount; }
 	const ConvexVolume* getConvexVolumes() const { return m_volumes; }
 	void addConvexVolume(const float* verts, const int nverts,
-						 const float minh, const float maxh, unsigned char area);
+						 const float minh, const float maxh, unsigned short flags, unsigned char area);
 	void deleteConvexVolume(int i);
 	void drawConvexVolumes(struct duDebugDraw* dd, const float* offset, bool hilight = false);
 	///@}

src/thirdparty/recast/Recast/Include/Recast.h

@@ -364,6 +364,7 @@ struct rcCompactHeightfield
 	rcCompactCell* cells;		///< Array of cells. [Size: #width*#height]
 	rcCompactSpan* spans;		///< Array of spans. [Size: #spanCount]
 	unsigned short* dist;		///< Array containing border distance data. [Size: #spanCount]
+	unsigned short* flags;		///< Array containing flags data. [Size: #spanCount]
 	unsigned char* areas;		///< Array containing area id data. [Size: #spanCount]
 	
 private:
@@ -414,10 +415,11 @@ private:
 struct rcContour
 {
 	int* verts;			///< Simplified contour vertex and connection data. [Size: 4 * #nverts]
-	int nverts;			///< The number of vertices in the simplified contour. 
 	int* rverts;		///< Raw contour vertex and connection data. [Size: 4 * #nrverts]
+	int nverts;			///< The number of vertices in the simplified contour. 
 	int nrverts;		///< The number of vertices in the raw contour. 
 	unsigned short reg;	///< The region id of the contour.
+	unsigned short flags;///< The flags of the contour.
 	unsigned char area;	///< The area id of the contour.
 };
 
@@ -939,9 +941,11 @@ bool rcMedianFilterWalkableArea(rcContext* ctx, rcCompactHeightfield& chf);
 /// @param[in,out]	ctx		The build context to use during the operation.
 /// @param[in]		bmin	The minimum of the bounding box. [(x, y, z)]
 /// @param[in]		bmax	The maximum of the bounding box. [(x, y, z)]
+/// @param[in]		flags	The flags to apply. [Limit: <= #RC_WALKABLE_AREA]
 /// @param[in]		areaId	The area id to apply. [Limit: <= #RC_WALKABLE_AREA]
 /// @param[in,out]	chf		A populated compact heightfield.
-void rcMarkBoxArea(rcContext* ctx, const float* bmin, const float* bmax, unsigned char areaId,
+void rcMarkBoxArea(rcContext* ctx, const float* bmin, const float* bmax,
+				   unsigned short flags, unsigned char areaId,
 				   rcCompactHeightfield& chf);
 
 /// Applies the area id to the all spans within the specified convex polygon. 
@@ -951,10 +955,12 @@ void rcMarkBoxArea(rcContext* ctx, const float* bmin, const float* bmax, unsigne
 /// @param[in]		nverts	The number of vertices in the polygon.
 /// @param[in]		hmin	The height of the base of the polygon.
 /// @param[in]		hmax	The height of the top of the polygon.
+/// @param[in]		flags	The flags to apply. [Limit: <= #RC_WALKABLE_AREA]
 /// @param[in]		areaId	The area id to apply. [Limit: <= #RC_WALKABLE_AREA]
 /// @param[in,out]	chf		A populated compact heightfield.
 void rcMarkConvexPolyArea(rcContext* ctx, const float* verts, const int nverts,
-						  const float hmin, const float hmax, unsigned char areaId,
+						  const float hmin, const float hmax, 
+						  unsigned short flags, unsigned char areaId,
 						  rcCompactHeightfield& chf);
 
 /// Helper function to offset voncex polygons for rcMarkConvexPolyArea.

src/thirdparty/recast/Recast/Source/Recast.cpp

@@ -135,6 +135,7 @@ rcCompactHeightfield::rcCompactHeightfield()
 , cells()
 , spans()
 , dist()
+, flags()
 , areas()
 {
 }
@@ -144,6 +145,7 @@ rcCompactHeightfield::~rcCompactHeightfield()
 	rdFree(cells);
 	rdFree(spans);
 	rdFree(dist);
+	rdFree(flags);
 	rdFree(areas);
 }
 
@@ -429,6 +431,13 @@ bool rcBuildCompactHeightfield(rcContext* context, const int walkableHeight, con
 		return false;
 	}
 	memset(compactHeightfield.spans, 0, sizeof(rcCompactSpan) * spanCount);
+	compactHeightfield.flags = (unsigned short*)rdAlloc(sizeof(unsigned short) * spanCount, RD_ALLOC_PERM);
+	if (!compactHeightfield.flags)
+	{
+		context->log(RC_LOG_ERROR, "rcBuildCompactHeightfield: Out of memory 'chf.flags' (%d)", spanCount);
+		return false;
+	}
+	memset(compactHeightfield.flags, RC_NULL_AREA, sizeof(unsigned short) * spanCount);
 	compactHeightfield.areas = (unsigned char*)rdAlloc(sizeof(unsigned char) * spanCount, RD_ALLOC_PERM);
 	if (!compactHeightfield.areas)
 	{

src/thirdparty/recast/Recast/Source/RecastArea.cpp

@@ -307,7 +307,8 @@ bool rcMedianFilterWalkableArea(rcContext* ctx, rcCompactHeightfield& chf)
 /// The value of spacial parameters are in world units.
 /// 
 /// @see rcCompactHeightfield, rcMedianFilterWalkableArea
-void rcMarkBoxArea(rcContext* ctx, const float* bmin, const float* bmax, unsigned char areaId,
+void rcMarkBoxArea(rcContext* ctx, const float* bmin, const float* bmax,
+				   unsigned short flags, unsigned char areaId,
 				   rcCompactHeightfield& chf)
 {
 	rdAssert(ctx);
@@ -342,7 +343,10 @@ void rcMarkBoxArea(rcContext* ctx, const float* bmin, const float* bmax, unsigne
 				if ((int)s.z >= minz && (int)s.z <= maxz)
 				{
 					if (chf.areas[i] != RC_NULL_AREA)
+					{
+						chf.flags[i] = flags;
 						chf.areas[i] = areaId;
+					}
 				}
 			}
 		}
@@ -373,7 +377,8 @@ static int pointInPoly(int nvert, const float* verts, const float* p) // todo(am
 /// 
 /// @see rcCompactHeightfield, rcMedianFilterWalkableArea
 void rcMarkConvexPolyArea(rcContext* ctx, const float* verts, const int nverts,
-						  const float hmin, const float hmax, unsigned char areaId,
+						  const float hmin, const float hmax,
+						  unsigned short flags, unsigned char areaId,
 						  rcCompactHeightfield& chf)
 {
 	rdAssert(ctx);
@@ -429,6 +434,7 @@ void rcMarkConvexPolyArea(rcContext* ctx, const float* verts, const int nverts,
 
 					if (pointInPoly(nverts, verts, p))
 					{
+						chf.flags[i] = flags;
 						chf.areas[i] = areaId;
 					}
 				}

src/thirdparty/recast/Recast/Source/RecastContour.cpp

@@ -911,10 +911,11 @@ bool rcBuildContours(rcContext* ctx, const rcCompactHeightfield& chf,
 					flags[i] = 0;
 					continue;
 				}
-				const unsigned short reg = chf.spans[i].reg;
-				if (!reg || (reg & RC_BORDER_REG))
+				const unsigned short chfReg = chf.spans[i].reg;
+				if (!chfReg || (chfReg & RC_BORDER_REG))
 					continue;
-				const unsigned char area = chf.areas[i];
+				const unsigned short chfFlags = chf.flags[i];
+				const unsigned char chfArea = chf.areas[i];
 				
 				verts.clear();
 				simplified.clear();
@@ -993,8 +994,9 @@ bool rcBuildContours(rcContext* ctx, const rcCompactHeightfield& chf,
 						}
 					}
 					
-					cont->reg = reg;
-					cont->area = area;
+					cont->reg = chfReg;
+					cont->flags = chfFlags;
+					cont->area = chfArea;
 				}
 			}
 		}

src/thirdparty/recast/Recast/Source/RecastMesh.cpp

@@ -1104,6 +1104,12 @@ bool rcBuildPolyMesh(rcContext* ctx, rcContourSet& cset, const int nvp, rcPolyMe
 		ctx->log(RC_LOG_ERROR, "rcBuildPolyMesh: Out of memory 'mesh.regs' (%d).", maxTris);
 		return false;
 	}
+	mesh.flags = (unsigned short*)rdAlloc(sizeof(unsigned short)*maxTris, RD_ALLOC_PERM);
+	if (!mesh.flags)
+	{
+		ctx->log(RC_LOG_ERROR, "rcBuildPolyMesh: Out of memory 'mesh.flags' (%d).", maxTris);
+		return false;
+	}
 	mesh.areas = (unsigned char*)rdAlloc(sizeof(unsigned char)*maxTris, RD_ALLOC_PERM);
 	if (!mesh.areas)
 	{
@@ -1125,6 +1131,7 @@ bool rcBuildPolyMesh(rcContext* ctx, rcContourSet& cset, const int nvp, rcPolyMe
 	memset(mesh.verts, 0, sizeof(unsigned short)*maxVertices*3);
 	memset(mesh.polys, 0xff, sizeof(unsigned short)*maxTris*nvp*2);
 	memset(mesh.regs, 0, sizeof(unsigned short)*maxTris);
+	memset(mesh.flags, 0, sizeof(unsigned short)*maxTris);
 	memset(mesh.areas, 0, sizeof(unsigned char)*maxTris);
 	
 	rdScopedDelete<int> nextVert((int*)rdAlloc(sizeof(int)*maxVertices, RD_ALLOC_TEMP));
@@ -1279,6 +1286,7 @@ bool rcBuildPolyMesh(rcContext* ctx, rcContourSet& cset, const int nvp, rcPolyMe
 			for (int k = 0; k < nvp; ++k)
 				p[k] = q[k];
 			mesh.regs[mesh.npolys] = cont.reg;
+			mesh.flags[mesh.npolys] = cont.flags;
 			mesh.areas[mesh.npolys] = cont.area;
 			mesh.npolys++;
 			if (mesh.npolys > maxTris)
@@ -1390,15 +1398,6 @@ bool rcBuildPolyMesh(rcContext* ctx, rcContourSet& cset, const int nvp, rcPolyMe
 
 		mesh.surfa[i] = (unsigned short)rdMathRoundf(polyArea*RC_POLY_SURFAREA_QUANT_FACTOR);
 	}
-
-	// Just allocate the mesh flags array. The user is responsible to fill it.
-	mesh.flags = (unsigned short*)rdAlloc(sizeof(unsigned short)*mesh.npolys, RD_ALLOC_PERM);
-	if (!mesh.flags)
-	{
-		ctx->log(RC_LOG_ERROR, "rcBuildPolyMesh: Out of memory 'mesh.flags' (%d).", mesh.npolys);
-		return false;
-	}
-	memset(mesh.flags, 0, sizeof(unsigned short) * mesh.npolys);
 	
 	if (mesh.nverts > 0xffff)
 	{

src/thirdparty/recast/Recast/Source/RecastRegion.cpp

@@ -529,7 +529,7 @@ struct rcRegion
 	
 	int spanCount;					// Number of spans belonging to this region
 	unsigned short id;				// ID of the region
-	unsigned char areaType;			// Are type.
+	unsigned char areaType;			// Area type.
 	bool remap;
 	bool visited;
 	bool overlap;