Recast: implement masking in raycast algorithm

Allow caller to filter out what to collide with. For the editor cursor, we want to test on everything (including triggers). For the traverse link algorithm, we only want to test on clip brushes or world geometry as traversing through a trigger volume is valid.

src/naveditor/ConvexVolumeTool.cpp

@@ -382,7 +382,7 @@ void ShapeVolumeTool::handleRenderOverlay(double* /*proj*/, double* /*model*/, i
 	if (!m_npts)
 	{
 		ImGui_RenderText(ImGuiTextAlign_e::kAlignLeft,
-			ImVec2(280, 40), ImVec4(1.0f,1.0f,1.0f,0.75f), "LMB: Create new shape.  SHIFT+LMB: Delete existing shape (click inside a shape).");
+			ImVec2(280, 40), ImVec4(1.0f,1.0f,1.0f,0.75f), "LMB: Create new shape.  SHIFT+LMB: Delete existing shape (click on a shape).");
 	}
 	else
 	{

src/naveditor/Editor.cpp

@@ -555,6 +555,11 @@ static bool polyEdgeFaceAgainst(const float* v1, const float* v2, const float* n
 	return (rdVdot2D(delta, n1) >= 0 && rdVdot2D(delta, n2) < 0);
 }
 
+// NOTE: we don't want to collide with trigger area's as this would otherwise
+// prevent a link from happening between 2 valid slabs that intersect with
+// something like a door or action trigger.
+static const int TRAVERSE_LINK_TRACE_MASK = TRACE_WORLD|TRACE_CLIP;
+
 static bool traverseLinkOffsetIntersectsGeom(const InputGeom* geom, const float* basePos, const float* offsetPos)
 {
 	// We need to fire a raycast from out initial
@@ -577,8 +582,8 @@ static bool traverseLinkOffsetIntersectsGeom(const InputGeom* geom, const float*
 	// Otherwise we create links between a mesh
 	// inside and outside an object, causing the
 	// ai to traverse inside of it.
-	if (geom->raycastMesh(basePos, offsetPos) ||
-		geom->raycastMesh(offsetPos, basePos))
+	if (geom->raycastMesh(basePos, offsetPos, TRAVERSE_LINK_TRACE_MASK) ||
+		geom->raycastMesh(offsetPos, basePos, TRAVERSE_LINK_TRACE_MASK))
 		return true;
 
 	return false;
@@ -694,8 +699,8 @@ static bool traverseLinkInLOS(void* userData, const float* lowPos, const float*
 	// won't be any navmesh on the higher pos in the first place.
 	// Its still possible there's something blocking on the lower
 	// pos' side, but this is a lot less likely to happen.
-	if (geom->raycastMesh(targetRayPos, lowPos) ||
-		geom->raycastMesh(lowPos, targetRayPos))
+	if (geom->raycastMesh(targetRayPos, lowPos, TRAVERSE_LINK_TRACE_MASK) ||
+		geom->raycastMesh(lowPos, targetRayPos, TRAVERSE_LINK_TRACE_MASK))
 		return false;
 
 	return true;

src/naveditor/InputGeom.cpp

@@ -508,7 +508,7 @@ bool InputGeom::saveGeomSet(const BuildSettings* settings)
 	return true;
 }
 
-bool InputGeom::raycastMesh(const float* src, const float* dst, float* tmin) const
+bool InputGeom::raycastMesh(const float* src, const float* dst, const unsigned int mask, float* tmin) const
 {
 	// Prune hit ray.
 	float btmin = 0, btmax = 1;
@@ -518,6 +518,9 @@ bool InputGeom::raycastMesh(const float* src, const float* dst, float* tmin) con
 	bool hit = false;
 	const int nvol = m_volumeCount;
 
+	const bool traceClip = mask & TRACE_CLIP;
+	const bool traceTrigger = mask & TRACE_TRIGGER;
+
 	float isectTmin = 1.0f;
 
 	for (int i = 0; i < nvol; i++)
@@ -526,8 +529,11 @@ bool InputGeom::raycastMesh(const float* src, const float* dst, float* tmin) con
 		float tsmin = 0.0f, tsmax = 1.0f;
 		bool isect = false;
 
-		if (vol.area != RC_NULL_AREA)
-			continue; // Clip brushes only.
+		if (vol.area == RC_NULL_AREA && !traceClip)
+			continue;
+
+		if (vol.area == DT_POLYAREA_TRIGGER && !traceTrigger)
+			continue;
 
 		if (vol.type == VOLUME_BOX)
 		{
@@ -566,6 +572,11 @@ bool InputGeom::raycastMesh(const float* src, const float* dst, float* tmin) con
 		return true;
 	}
 
+	const bool traceWorld = mask & TRACE_WORLD;
+
+	if (!traceWorld)
+		return false;
+
 	float p[2], q[2];
 	p[0] = src[0] + (dst[0]-src[0]) * btmin;
 	p[1] = src[1] + (dst[1]-src[1]) * btmin;

src/naveditor/include/InputGeom.h

@@ -29,6 +29,14 @@ enum VolumeType : unsigned char
 	VOLUME_CONVEX
 };
 
+enum TraceMask : unsigned int
+{
+	TRACE_WORLD   = 1<<0, // The imported world geometry.
+	TRACE_CLIP    = 1<<1, // Clip brushes.
+	TRACE_TRIGGER = 1<<2, // Trigger brushes.
+	TRACE_ALL = 0xffffffff
+};
+
 static const int MAX_SHAPEVOL_PTS = 12;
 struct ShapeVolume
 {
@@ -142,7 +150,7 @@ public:
 
 	const rcChunkyTriMesh* getChunkyMesh() const { return m_chunkyMesh; }
 	const BuildSettings* getBuildSettings() const { return m_hasBuildSettings ? &m_buildSettings : 0; }
-	bool raycastMesh(const float* src, const float* dst, float* tmin = nullptr) const;
+	bool raycastMesh(const float* src, const float* dst, const unsigned int mask, float* tmin = nullptr) const;
 
 	/// @name Off-Mesh connections.
 	///@{

src/naveditor/main.cpp

@@ -771,7 +771,7 @@ int not_main(int argc, char** argv)
 		if (processHitTest && geom && editor)
 		{
 			float hitTime;
-			bool hit = geom->raycastMesh(rayStart, rayEnd, &hitTime);
+			bool hit = geom->raycastMesh(rayStart, rayEnd, TRACE_ALL, &hitTime);
 			
 			if (hit)
 			{