Recast: fix heap buffer overflow in partitioning code

Fixes https://github.com/recastnavigation/recastnavigation/issues/317. This can happen when we mave multiple spans in the X row. So far this crash only occured in Capitol City in Apex Legend's Worlds Edge map, where there are internal polygons in high rise buildings with 10 or more floors equally placed on top of another.

src/thirdparty/recast/Recast/Source/RecastLayers.cpp

@@ -94,7 +94,7 @@ bool rcBuildHeightfieldLayers(rcContext* ctx, rcCompactHeightfield& chf,
 	const int w = chf.width;
 	const int h = chf.height;
 	
-	rdScopedDelete<unsigned char> srcReg((unsigned char*)rdAlloc(sizeof(unsigned char)*chf.spanCount, RD_ALLOC_TEMP));
+	rdScopedDelete<unsigned char> srcReg(chf.spanCount);
 	if (!srcReg)
 	{
 		ctx->log(RC_LOG_ERROR, "rcBuildHeightfieldLayers: Out of memory 'srcReg' (%d).", chf.spanCount);
@@ -103,7 +103,7 @@ bool rcBuildHeightfieldLayers(rcContext* ctx, rcCompactHeightfield& chf,
 	memset(srcReg,0xff,sizeof(unsigned char)*chf.spanCount);
 	
 	const int nsweeps = chf.width;
-	rdScopedDelete<rcLayerSweepSpan> sweeps((rcLayerSweepSpan*)rdAlloc(sizeof(rcLayerSweepSpan)*nsweeps, RD_ALLOC_TEMP));
+	rdScopedDelete<rcLayerSweepSpan> sweeps(nsweeps);
 	if (!sweeps)
 	{
 		ctx->log(RC_LOG_ERROR, "rcBuildHeightfieldLayers: Out of memory 'sweeps' (%d).", nsweeps);
@@ -144,8 +144,18 @@ bool rcBuildHeightfieldLayers(rcContext* ctx, rcCompactHeightfield& chf,
 				if (sid == 0xff)
 				{
 					sid = sweepId++;
-					sweeps[sid].nei = 0xff;
+
-					sweeps[sid].ns = 0;
+					// If we have multiple spans in the X row, we might need more memory than initially allocated.
+					if (sweeps.grow(sid+1))
+					{
+						sweeps[sid].nei = 0xff;
+						sweeps[sid].ns = 0;
+					}
+					else
+					{
+						ctx->log(RC_LOG_ERROR, "rcBuildHeightfieldLayers: Out of memory 'sweeps.grow(%d)'.", sid+1);
+						return false;
+					}
 				}
 				
 				// -y

src/thirdparty/recast/Recast/Source/RecastRegion.cpp

@@ -1369,7 +1369,7 @@ bool rcBuildRegionsMonotone(rcContext* ctx, rcCompactHeightfield& chf,
 	memset(srcReg,0,sizeof(unsigned short)*chf.spanCount);
 
 	const int nsweeps = rdMax(chf.width,chf.height);
-	rdScopedDelete<rcSweepSpan> sweeps((rcSweepSpan*)rdAlloc(sizeof(rcSweepSpan)*nsweeps, RD_ALLOC_TEMP));
+	rdScopedDelete<rcSweepSpan> sweeps(nsweeps);
 	if (!sweeps)
 	{
 		ctx->log(RC_LOG_ERROR, "rcBuildRegionsMonotone: Out of memory 'sweeps' (%d).", nsweeps);
@@ -1426,17 +1426,18 @@ bool rcBuildRegionsMonotone(rcContext* ctx, rcCompactHeightfield& chf,
 				{
 					previd = rid++;
 
-					// todo(amos): figure out why this happens on very complex and large
+					// If we have multiple spans in the X row, we might need more memory than initially allocated.
-					// input geometry.
+					if (sweeps.grow(previd+1))
-					if (previd >= nsweeps)
 					{
-						ctx->log(RC_LOG_ERROR, "rcBuildLayerRegions: Sweep index out of bounds: previd (%d), nsweeps (%d).", previd, nsweeps);
+						sweeps[previd].rid = previd;
+						sweeps[previd].ns = 0;
+						sweeps[previd].nei = 0;
+					}
+					else
+					{
+						ctx->log(RC_LOG_ERROR, "rcBuildRegionsMonotone: Out of memory 'sweeps.grow(%d)'.", previd+1);
 						return false;
 					}
-
-					sweeps[previd].rid = previd;
-					sweeps[previd].ns = 0;
-					sweeps[previd].nei = 0;
 				}
 
 				// -y
@@ -1744,17 +1745,18 @@ bool rcBuildLayerRegions(rcContext* ctx, rcCompactHeightfield& chf,
 				{
 					previd = rid++;
 
-					// todo(amos): figure out why this happens on very complex and large
+					// If we have multiple spans in the X row, we might need more memory than initially allocated.
-					// input geometry.
+					if (sweeps.grow(previd+1))
-					if (previd >= nsweeps)
 					{
-						ctx->log(RC_LOG_ERROR, "rcBuildLayerRegions: Sweep index out of bounds: previd (%d), nsweeps (%d).", previd, nsweeps);
+						sweeps[previd].rid = previd;
+						sweeps[previd].ns = 0;
+						sweeps[previd].nei = 0;
+					}
+					else
+					{
+						ctx->log(RC_LOG_ERROR, "rcBuildLayerRegions: Out of memory 'sweeps.grow(%d)'.", previd+1);
 						return false;
 					}
-
-					sweeps[previd].rid = previd;
-					sweeps[previd].ns = 0;
-					sweeps[previd].nei = 0;
 				}
 				
 				// -y

src/thirdparty/recast/Shared/Include/SharedAlloc.h

@@ -325,20 +325,29 @@ public:
 /// @note This class is rarely if ever used by the end user.
 template<class T> class rdScopedDelete
 {
-	T* ptr;
+	T* m_ptr;
+	rdSizeType m_size;
 public:
 
 	/// Constructs an instance with a null pointer.
-	inline rdScopedDelete() : ptr(0) {}
+	inline rdScopedDelete() : m_ptr(0), m_size(0) {}
+	inline rdScopedDelete(const rdSizeType n) { m_ptr = (T*)rdAlloc(sizeof(T)*n, RD_ALLOC_TEMP); m_size = n; }
 
 	/// Constructs an instance with the specified pointer.
-	///  @param[in]		p	An pointer to an allocated array.
+	///  @param[in]		p	A pointer to an allocated array.
-	inline rdScopedDelete(T* p) : ptr(p) {}
+	inline rdScopedDelete(T* p) : m_ptr(p), m_size(0) {}
-	inline ~rdScopedDelete() { rdFree(ptr); }
+	inline ~rdScopedDelete() { rdFree(m_ptr); }
+
+	/// Gets the number of elements.
+	inline rdSizeType size() const { return m_size; }
+
+	/// Grow and move existing memory.
+	///  @param[in]		n	New element count.
+	inline bool grow(const rdSizeType n);
 
 	/// The root array pointer.
 	///  @return The root array pointer.
-	inline operator T*() { return ptr; }
+	inline operator T*() { return m_ptr; }
 	
 private:
 	// Explicitly disabled copy constructor and copy assignment operator.
@@ -346,4 +355,21 @@ private:
 	rdScopedDelete& operator=(const rdScopedDelete&);
 };
 
+template<class T>
+inline bool rdScopedDelete<T>::grow(const rdSizeType n)
+{
+	if (n > m_size)
+	{
+		T* newData = (T*)rdAlloc(sizeof(T)*n, RD_ALLOC_TEMP);
+		if (n && newData) {
+			memcpy(newData, m_ptr, sizeof(T)*n);
+		}
+		rdFree(m_ptr);
+		m_ptr = newData;
+		m_size = n;
+	}
+
+	return (m_ptr != 0);
+}
+
 #endif // RECASTDETOURALLOC_H