Actually, lets just fix FixedHypot instead.

Now FixedHypot uses the code from R_PointToDist2, and R_PointToDist2 just calls FixedHypot.

Ultimately, this branch was intended to get rid of a redundant way to retrieve distance and replace it with the one that was actually good at its job. So consolidating FixedHypot and R_PointToDist2 together is just an extension of that.

src/lua_baselib.c

@@ -238,8 +238,8 @@ static int lib_pAproxDistance(lua_State *L)
 	fixed_t dx = luaL_checkfixed(L, 1);
 	fixed_t dy = luaL_checkfixed(L, 2);
 	//HUDSAFE
-	LUA_Deprecated(L, "P_AproxDistance", "R_PointToDist2");
-	lua_pushfixed(L, R_PointToDist2(0, 0, dx, dy));
+	LUA_Deprecated(L, "P_AproxDistance", "FixedHypot");
+	lua_pushfixed(L, FixedHypot(dx, dy));
 	return 1;
 }
 

src/m_fixed.c

@@ -18,8 +18,10 @@
  #define HAVE_SQRTF
  #endif
 #endif
+
 #include "doomdef.h"
 #include "m_fixed.h"
+#include "tables.h" // ANGLETOFINESHIFT
 
 #ifdef __USE_C_FIXEDMUL__
 
@@ -105,20 +107,34 @@ fixed_t FixedSqrt(fixed_t x)
 
 fixed_t FixedHypot(fixed_t x, fixed_t y)
 {
-	fixed_t ax, yx, yx2, yx1;
-	if (abs(y) > abs(x)) // |y|>|x|
+	// Moved the code from R_PointToDist2 to here,
+	// since R_PointToDist2 did the same thing,
+	// except less prone to overflowing.
+
+	angle_t angle;
+	fixed_t dist;
+
+	x = abs(x);
+	y = abs(y);
+
+	if (y > x)
 	{
-		ax = abs(y); // |y| => ax
-		yx = FixedDiv(x, y); // (x/y)
+		fixed_t temp;
+
+		temp = x;
+		x = y;
+		y = temp;
 	}
-	else // |x|>|y|
-	{
-		ax = abs(x); // |x| => ax
-		yx = FixedDiv(y, x); // (x/y)
-	}
-	yx2 = FixedMul(yx, yx); // (x/y)^2
-	yx1 = FixedSqrt(1 * FRACUNIT + yx2); // (1 + (x/y)^2)^1/2
-	return FixedMul(ax, yx1); // |x|*((1 + (x/y)^2)^1/2)
+
+	if (!y)
+		return x;
+
+	angle = (tantoangle[FixedDiv(y, x)>>DBITS] + ANGLE_90) >> ANGLETOFINESHIFT;
+
+	// use as cosine
+	dist = FixedDiv(x, FINESINE(angle));
+
+	return dist;
 }
 
 vector2_t *FV2_Load(vector2_t *vec, fixed_t x, fixed_t y)

src/p_maputl.h

@@ -41,7 +41,7 @@ typedef boolean (*traverser_t)(intercept_t *in);
 boolean P_PathTraverse(fixed_t px1, fixed_t py1, fixed_t px2, fixed_t py2,
 	INT32 pflags, traverser_t ptrav);
 
-#define P_AproxDistance(dx, dy) R_PointToDist2(0, 0, dx, dy)
+#define P_AproxDistance(dx, dy) FixedHypot(dx, dy)
 void P_ClosestPointOnLine(fixed_t x, fixed_t y, line_t *line, vertex_t *result);
 void P_ClosestPointOnLine3D(fixed_t x, fixed_t y, fixed_t z, line_t *line, vertex_t *result);
 INT32 P_PointOnLineSide(fixed_t x, fixed_t y, line_t *line);

src/r_main.c

@@ -393,29 +393,7 @@ angle_t R_PointToAngle2(fixed_t pviewx, fixed_t pviewy, fixed_t x, fixed_t y)
 
 fixed_t R_PointToDist2(fixed_t px2, fixed_t py2, fixed_t px1, fixed_t py1)
 {
-	angle_t angle;
-	fixed_t dx, dy, dist;
-
-	dx = abs(px1 - px2);
-	dy = abs(py1 - py2);
-
-	if (dy > dx)
-	{
-		fixed_t temp;
-
-		temp = dx;
-		dx = dy;
-		dy = temp;
-	}
-	if (!dy)
-		return dx;
-
-	angle = (tantoangle[FixedDiv(dy, dx)>>DBITS] + ANGLE_90) >> ANGLETOFINESHIFT;
-
-	// use as cosine
-	dist = FixedDiv(dx, FINESINE(angle));
-
-	return dist;
+	return FixedHypot(px1 - px2, py1 - py2);
 }
 
 // Little extra utility. Works in the same way as R_PointToAngle2