Recast: implement off-mesh connection yaw angles and reference positions

Used by the engine for AI wall running, but also other AI logic like move direction. This probably is some cached ref pos + yaw to save on computation in the runtime, and probably used to lerp the AI to the correct direction before jumping. Current implementation seems very close to original navs now, though, the original navs appear to have some yaw angles being rotated a bit, probably manual adjustments to make the AI face a wall that isn't perpendicular.

Also implemented debug drawing for the new ref positions, and made the start circle of the off-mesh connection red, and the end position green to make it easy what is what in bidirectional connections.

src/naveditor/Editor_TileMesh.cpp

@@ -1177,6 +1177,8 @@ unsigned char* Editor_TileMesh::buildTileMesh(const int tx, const int ty, const
 		params.offMeshConAreas = m_geom->getOffMeshConnectionAreas();
 		params.offMeshConFlags = m_geom->getOffMeshConnectionFlags();
 		params.offMeshConUserID = m_geom->getOffMeshConnectionId();
+		params.offMeshConRefPos = m_geom->getOffMeshConnectionRefPos();
+		params.offMeshConRefYaw = m_geom->getOffMeshConnectionRefYaws();
 		params.offMeshConCount = m_geom->getOffMeshConnectionCount();
 		params.walkableHeight = m_agentHeight;
 		params.walkableRadius = m_agentRadius;

src/naveditor/GameUtils.cpp

@@ -59,7 +59,7 @@ void patchTileGame(dtMeshTile* t)
 	for (size_t i = 0; i < t->header->detailVertCount * 3; i += 3)
 		coordGameSwap(t->detailVerts + i);
 	for (size_t i = 0; i < t->header->polyCount; i++)
-		coordGameSwap(t->polys[i].org);
+		coordGameSwap(t->polys[i].center);
 	//might be wrong because of coord change might break tree layout
 	for (size_t i = 0; i < t->header->bvNodeCount; i++)
 	{
@@ -70,7 +70,7 @@ void patchTileGame(dtMeshTile* t)
 	{
 		coordGameSwap(t->offMeshCons[i].pos);
 		coordGameSwap(t->offMeshCons[i].pos + 3);
-		coordGameSwap(t->offMeshCons[i].unk);
+		coordGameSwap(t->offMeshCons[i].refPos);
 	}
 }
 void unpatchTileGame(dtMeshTile* t)
@@ -83,7 +83,7 @@ void unpatchTileGame(dtMeshTile* t)
 	for (size_t i = 0; i < t->header->detailVertCount * 3; i += 3)
 		coordGameUnswap(t->detailVerts + i);
 	for (size_t i = 0; i < t->header->polyCount; i++)
-		coordGameUnswap(t->polys[i].org);
+		coordGameUnswap(t->polys[i].center);
 	//might be wrong because of coord change might break tree layout
 	for (size_t i = 0; i < t->header->bvNodeCount; i++)
 	{
@@ -94,7 +94,7 @@ void unpatchTileGame(dtMeshTile* t)
 	{
 		coordGameUnswap(t->offMeshCons[i].pos);
 		coordGameUnswap(t->offMeshCons[i].pos + 3);
-		coordGameUnswap(t->offMeshCons[i].unk);
+		coordGameUnswap(t->offMeshCons[i].refPos);
 	}
 }
 
@@ -164,11 +164,13 @@ void buildLinkTable(dtNavMesh* mesh, LinkTableData& data)
 }
 void setReachable(std::vector<int>& data, int count, int id1, int id2, bool value)
 {
-	int w = ((count + 31) / 32);
-	auto& cell = data[id1 * w + id2 / 32];
-	uint32_t value_mask = ~(1 << (id2 & 0x1f));
+	const int w = ((count + 31) / 32);
+	const unsigned int valueMask = ~(1 << (id2 & 0x1f));
+
+	int& cell = data[id1 * w + id2 / 32];
+
 	if (!value)
-		cell = (cell & value_mask);
+		cell = (cell & valueMask);
 	else
-		cell = (cell & value_mask) | (1 << (id2 & 0x1f));
-}
+		cell = (cell & valueMask) | (1 << (id2 & 0x1f));
+}

src/naveditor/InputGeom.cpp

@@ -511,25 +511,35 @@ void InputGeom::addOffMeshConnection(const float* spos, const float* epos, const
 									 unsigned char bidir, unsigned char area, unsigned short flags)
 {
 	if (m_offMeshConCount >= MAX_OFFMESH_CONNECTIONS) return;
-	float* v = &m_offMeshConVerts[m_offMeshConCount*3*2];
+	float* refs = &m_offMeshConRefPos[m_offMeshConCount*3];
+	float* verts = &m_offMeshConVerts[m_offMeshConCount*3*2];
+	float yaw = dtCalcOffMeshRefYaw(spos, epos);
+
+	dtCalcOffMeshRefPos(spos, yaw, DT_OFFMESH_CON_REFPOS_OFFSET, refs);
+
 	m_offMeshConRads[m_offMeshConCount] = rad;
+	m_offMeshConRefYaws[m_offMeshConCount] = yaw;
 	m_offMeshConDirs[m_offMeshConCount] = bidir;
 	m_offMeshConAreas[m_offMeshConCount] = area;
 	m_offMeshConFlags[m_offMeshConCount] = flags;
 	m_offMeshConId[m_offMeshConCount] = 1000 + m_offMeshConCount;
-	rcVcopy(&v[0], spos);
-	rcVcopy(&v[3], epos);
+	rcVcopy(&verts[0], spos);
+	rcVcopy(&verts[3], epos);
 	m_offMeshConCount++;
 }
 
 void InputGeom::deleteOffMeshConnection(int i)
 {
 	m_offMeshConCount--;
-	float* src = &m_offMeshConVerts[m_offMeshConCount*3*2];
-	float* dst = &m_offMeshConVerts[i*3*2];
-	rcVcopy(&dst[0], &src[0]);
-	rcVcopy(&dst[3], &src[3]);
+	float* vertsSrc = &m_offMeshConVerts[m_offMeshConCount*3*2];
+	float* vertsDst = &m_offMeshConVerts[i*3*2];
+	float* refSrc = &m_offMeshConRefPos[m_offMeshConCount*3];
+	float* refDst = &m_offMeshConRefPos[i*3];
+	rcVcopy(&vertsDst[0], &vertsSrc[0]);
+	rcVcopy(&vertsDst[3], &vertsSrc[3]);
+	rcVcopy(&refDst[0], &refSrc[0]);
 	m_offMeshConRads[i] = m_offMeshConRads[m_offMeshConCount];
+	m_offMeshConRefYaws[i] = m_offMeshConRefYaws[m_offMeshConCount];
 	m_offMeshConDirs[i] = m_offMeshConDirs[m_offMeshConCount];
 	m_offMeshConAreas[i] = m_offMeshConAreas[m_offMeshConCount];
 	m_offMeshConFlags[i] = m_offMeshConFlags[m_offMeshConCount];

src/naveditor/include/InputGeom.h

@@ -86,6 +86,8 @@ class InputGeom
 	unsigned char m_offMeshConAreas[MAX_OFFMESH_CONNECTIONS];
 	unsigned short m_offMeshConFlags[MAX_OFFMESH_CONNECTIONS];
 	unsigned int m_offMeshConId[MAX_OFFMESH_CONNECTIONS];
+	float m_offMeshConRefPos[MAX_OFFMESH_CONNECTIONS*3];
+	float m_offMeshConRefYaws[MAX_OFFMESH_CONNECTIONS];
 	int m_offMeshConCount;
 	///@}
 
@@ -126,6 +128,8 @@ public:
 	const unsigned char* getOffMeshConnectionAreas() const { return m_offMeshConAreas; }
 	const unsigned short* getOffMeshConnectionFlags() const { return m_offMeshConFlags; }
 	const unsigned int* getOffMeshConnectionId() const { return m_offMeshConId; }
+	const float* getOffMeshConnectionRefPos() const { return m_offMeshConRefPos; }
+	const float* getOffMeshConnectionRefYaws() const { return m_offMeshConRefYaws; }
 	void addOffMeshConnection(const float* spos, const float* epos, const float rad,
 							  unsigned char bidir, unsigned char area, unsigned short flags);
 	void deleteOffMeshConnection(int i);

src/thirdparty/recast/DebugUtils/Source/DetourDebugDraw.cpp

@@ -22,6 +22,15 @@
 #include "Detour/Include/DetourCommon.h"
 #include "Detour/Include/DetourNode.h"
 
+static void drawOffMeshConnectionRefPosition(duDebugDraw* dd, const dtOffMeshConnection* con)
+{
+	float refPosDir[3];
+	dtCalcOffMeshRefPos(con->refPos, con->refYaw, DT_OFFMESH_CON_REFPOS_OFFSET, refPosDir);
+
+	duAppendArrow(dd, con->refPos[0], con->refPos[1], con->refPos[2],
+		refPosDir[0], refPosDir[1], refPosDir[2], 0.f, 10.f, duRGBA(255, 255, 0, 255));
+}
+
 static void drawPolyBoundaries(duDebugDraw* dd, const dtMeshTile* tile,
 							   const unsigned int col, const float linew,
 							   bool inner)
@@ -190,7 +199,7 @@ static void drawMeshTile(duDebugDraw* dd, const dtNavMesh& mesh, const dtNavMesh
 
 			dd->vertex(vb[0],vb[1],vb[2], col);
 			dd->vertex(con->pos[3],con->pos[4],con->pos[5], col);
-			col2 = endSet ? col : duRGBA(220,32,16,196);
+			col2 = endSet ? col : duRGBA(32,220,16,196);
 			duAppendCircle(dd, con->pos[3],con->pos[4],con->pos[5]+5.0f, con->rad, col2);
 			
 			// End point vertices.
@@ -203,6 +212,9 @@ static void drawMeshTile(duDebugDraw* dd, const dtNavMesh& mesh, const dtNavMesh
 			// Connection arc.
 			duAppendArc(dd, con->pos[0],con->pos[1],con->pos[2], con->pos[3],con->pos[4],con->pos[5], 0.25f,
 						(con->flags & DT_OFFMESH_CON_BIDIR) ? 30.0f : 0.0f, 30.0f, col);
+
+			// Reference positions.
+			drawOffMeshConnectionRefPosition(dd, con);
 		}
 		dd->end();
 	}
@@ -443,6 +455,8 @@ void duDebugDrawNavMeshPoly(duDebugDraw* dd, const dtNavMesh& mesh, dtPolyRef re
 		// Connection arc.
 		duAppendArc(dd, con->pos[0],con->pos[1],con->pos[2], con->pos[3],con->pos[4],con->pos[5], 0.25f,
 					(con->flags & DT_OFFMESH_CON_BIDIR) ? 30.0f : 0.0f, 30.0f, c);
+
+		drawOffMeshConnectionRefPosition(dd, con);
 		
 		dd->end();
 	}

src/thirdparty/recast/Detour/Include/DetourNavMesh.h

@@ -93,6 +93,9 @@ static const unsigned int DT_NULL_LINK = 0xffffffff;
 /// A flag that indicates that an off-mesh connection can be traversed in both directions. (Is bidirectional.)
 static const unsigned int DT_OFFMESH_CON_BIDIR = 1;
 
+/// A value that determines the offset between the start pos and the ref pos in an off-mesh connection.
+static const float DT_OFFMESH_CON_REFPOS_OFFSET = 35.f;
+
 /// The maximum number of user defined area ids.
 /// @ingroup detour
 static const int DT_MAX_AREAS = 32; // <-- confirmed 32 see [r5apex_ds.exe + 0xf47dda] '-> test    [rcx+80h], ax'.
@@ -183,7 +186,9 @@ struct dtPoly
 	unsigned short disjointSetId;
 	unsigned short unk;						//IDK but looks filled
 	unsigned int unk1;						//!TODO: debug this if you ever find where this gets used in the engine..
-	float org[3];							// Seems to be used for AIN file generation (build from large navmesh).
+
+	/// The center of the polygon; see abstracted script function 'Navmesh_RandomPositions'.
+	float center[3];
 
 	/// Sets the user defined area id. [Limit: < #DT_MAX_AREAS]
 	inline void setArea(unsigned char a) { areaAndtype = (areaAndtype & 0xc0) | (a & 0x3f); }
@@ -255,10 +260,24 @@ struct dtOffMeshConnection
 	/// The id of the offmesh connection. (User assigned when the navigation mesh is built.)
 	unsigned int userId;
 
-	float unk[3];
-	float traverseYaw;
+	/// The reference position set to the start of the off-mesh connection with an offset of DT_OFFMESH_CON_REFPOS_OFFSET
+	float refPos[3]; // See [r5apex_ds + F114CF], [r5apex_ds + F11B42], [r5apex_ds + F12447].
+	/// The reference yaw angle set towards the end position of the off-mesh connection.
+	float refYaw;    // See [r5apex_ds + F11527], [r5apex_ds + F11F90], [r5apex_ds + F12836].
 };
 
+/// Calculates the yaw angle in an off-mesh connection.
+/// @param	spos[in]		The start position of the off mesh connection.
+/// @param	epos[in]		The end position of the off mesh connection.
+///								returns the yaw angle on the XY plane.
+extern float dtCalcOffMeshRefYaw(const float* spos, const float* epos);
+/// Calculates the ref position in an off-mesh connection.
+/// @param	spos[in]		The start position of the off mesh connection.
+/// @param	yaw[in]			The yaw angle of the off-mesh connection.
+/// @param	offset[in]		The desired offset from the start position.
+/// @param	res[in]			The output ref position.
+extern void dtCalcOffMeshRefPos(const float* spos, float yaw, float offset, float* res);
+
 /// Provides high level information related to a dtMeshTile object.
 /// @ingroup detour
 struct dtMeshHeader
@@ -337,7 +356,7 @@ private:
 
 /// Get flags for edge in detail triangle.
 /// @param	triFlags[in]		The flags for the triangle (last component of detail vertices above).
-/// @param	edgeIndex[in]		The index of the first vertex of the edge. For instance, if 0,
+/// @param	edgeIndex[in]		The index of the first vertex of the edge. For instance, if 0.
 ///								returns flags for edge AB.
 inline int dtGetDetailTriEdgeFlags(unsigned char triFlags, int edgeIndex)
 {

src/thirdparty/recast/Detour/Include/DetourNavMeshBuilder.h

@@ -69,9 +69,13 @@ struct dtNavMeshCreateParams
 	///
 	/// 0 = Travel only from endpoint A to endpoint B.<br/>
 	/// #DT_OFFMESH_CON_BIDIR = Bidirectional travel.
-	const unsigned char* offMeshConDir;	
+	const unsigned char* offMeshConDir;
 	/// The user defined ids of the off-mesh connection. [Size: #offMeshConCount]
 	const unsigned int* offMeshConUserID;
+	/// Off-mesh connection reference positions. [(x, y, z) * #offMeshConCount] [Unit: wu]
+	const float* offMeshConRefPos;
+	/// Off-mesh connection reference yaw. [Size: #offMeshConCount] [Unit: wu]
+	const float* offMeshConRefYaw;
 	/// The number of off-mesh connections. [Limit: >= 0]
 	int offMeshConCount;
 

src/thirdparty/recast/Detour/Source/DetourNavMesh.cpp

@@ -258,6 +258,7 @@ dtStatus dtNavMesh::init(const dtNavMeshParams* params)
 	// Only allow 31 salt bits, since the salt mask is calculated using 32bit uint and it will overflow.
 	m_saltBits = dtMin((unsigned int)31, 32 - m_tileBits - m_polyBits);
 
+	// NOTE[ AMOS ]: this check doesn't exist in R5!
 	if (m_saltBits < 10)
 		return DT_FAILURE | DT_INVALID_PARAM;
 #endif
@@ -1613,3 +1614,22 @@ dtStatus dtNavMesh::getPolyArea(dtPolyRef ref, unsigned char* resultArea) const
 	return DT_SUCCESS;
 }
 
+float dtCalcOffMeshRefYaw(const float* spos, const float* epos)
+{
+	float dx = epos[0] - spos[0];
+	float dy = epos[1] - spos[1];
+
+	// Amos: yaw on original r2 sp navs' seem to be of range [180, -180], might need to multiply this with (180.0f/DT_PI).
+	float yaw = dtMathAtan2f(dy, dx);
+	return yaw;
+}
+
+void dtCalcOffMeshRefPos(const float* spos, float yaw, float offset, float* res)
+{
+	float dx = offset * dtMathCosf(yaw);
+	float dy = offset * dtMathSinf(yaw);
+
+	res[0] = spos[0] + dx;
+	res[1] = spos[1] + dy;
+	res[2] = spos[2];
+}

src/thirdparty/recast/Detour/Source/DetourNavMeshBuilder.cpp

@@ -347,7 +347,7 @@ bool dtCreateNavMeshData(dtNavMeshCreateParams* params, unsigned char** outData,
 					offMeshConClass[i*2+0] = 0;
 			}
 
-			// Cound how many links should be allocated for off-mesh connections.
+			// Count how many links should be allocated for off-mesh connections.
 			if (offMeshConClass[i*2+0] == 0xff)
 				offMeshConLinkCount++;
 			if (offMeshConClass[i*2+1] == 0xff)
@@ -358,7 +358,7 @@ bool dtCreateNavMeshData(dtNavMeshCreateParams* params, unsigned char** outData,
 		}
 	}
 	
-	// Off-mesh connectionss are stored as polygons, adjust values.
+	// Off-mesh connections are stored as polygons, adjust values.
 	const int totPolyCount = params->polyCount + storedOffMeshConCount;
 	const int totVertCount = params->vertCount + storedOffMeshConCount*2;
 	
@@ -564,10 +564,10 @@ bool dtCreateNavMeshData(dtNavMeshCreateParams* params, unsigned char** outData,
 				// Normal connection
 				p->neis[j] = src[nvp+j]+1;
 			}
-			dtVadd(p->org, p->org, &navVerts[p->verts[j] * 3]);
+			dtVadd(p->center, p->center, &navVerts[p->verts[j] * 3]);
 			p->vertCount++;
 		}
-		dtVscale(p->org, p->org, 1 / (float)(p->vertCount));
+		dtVscale(p->center, p->center, 1 / (float)(p->vertCount));
 		src += nvp*2;
 	}
 	// Off-mesh connection vertices.
@@ -659,9 +659,12 @@ bool dtCreateNavMeshData(dtNavMeshCreateParams* params, unsigned char** outData,
 			con->poly = (unsigned short)(offMeshPolyBase + n);
 			// Copy connection end-points.
 			const float* endPts = &params->offMeshConVerts[i*2*3];
+			const float* refPos = &params->offMeshConRefPos[i*3];
 			dtVcopy(&con->pos[0], &endPts[0]);
 			dtVcopy(&con->pos[3], &endPts[3]);
+			dtVcopy(&con->refPos[0], &refPos[0]);
 			con->rad = params->offMeshConRad[i];
+			con->refYaw = params->offMeshConRefYaw[i];
 			con->flags = params->offMeshConDir[i] ? DT_OFFMESH_CON_BIDIR : 0;
 			con->side = offMeshConClass[i*2+1];
 			if (params->offMeshConUserID)

src/thirdparty/recast/DetourTileCache/Source/DetourTileCache.cpp

@@ -164,6 +164,8 @@ dtStatus dtTileCache::init(const dtTileCacheParams* params,
 	m_tileBits = dtIlog2(dtNextPow2((unsigned int)m_params.maxTiles));
 	// Only allow 31 salt bits, since the salt mask is calculated using 32bit uint and it will overflow.
 	m_saltBits = dtMin((unsigned int)31, 32 - m_tileBits);
+
+	// NOTE[ AMOS ]: this check doesn't exist in dtNavMesh::init for R5, so it probably also doesn't exist here!
 	if (m_saltBits < 10)
 		return DT_FAILURE | DT_INVALID_PARAM;