From dc02ff5654c455ebef3f3da9ce434d70879921e3 Mon Sep 17 00:00:00 2001
From: helixhorned <helixhorned@1a8010ca-5511-0410-912e-c29ae57300e0>
Date: Tue, 6 Sep 2011 17:46:17 +0000
Subject: [PATCH] Using this dynamic checking tool:
 http://embed.cs.utah.edu/ubc/, fix two issues with signed integer overflow.
 One is related to clipping and the other to Polymost mouse-picking for
 overlong walls. Many more remain especially in the 8-bit rendering code, but
 I expect many of them to be intended.

git-svn-id: https://svn.eduke32.com/eduke32@2009 1a8010ca-5511-0410-912e-c29ae57300e0
---
 polymer/eduke32/build/src/engine.c   | 10 ++++++----
 polymer/eduke32/build/src/polymost.c |  8 +++++---
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/polymer/eduke32/build/src/engine.c b/polymer/eduke32/build/src/engine.c
index fbc764154..351c31408 100644
--- a/polymer/eduke32/build/src/engine.c
+++ b/polymer/eduke32/build/src/engine.c
@@ -11895,11 +11895,15 @@ int32_t clipmove(vec3_t *pos, int16_t *sectnum,
         hitwall = raytrace(pos->x, pos->y, &intx, &inty);
         if (hitwall >= 0)
         {
+            uint64_t tempull;
+
             lx = clipit[hitwall].x2-clipit[hitwall].x1;
             ly = clipit[hitwall].y2-clipit[hitwall].y1;
-            tempint2 = lx*lx + ly*ly;
-            if (tempint2 > 0)
+            tempull = (int64_t)lx*(int64_t)lx + (int64_t)ly*(int64_t)ly;
+            if (tempull > 0 && tempull < INT32_MAX)
             {
+                tempint2 = (int32_t)tempull;
+
                 tempint1 = (goalx-intx)*lx + (goaly-inty)*ly;
 
                 if ((klabs(tempint1)>>11) < tempint2)
@@ -11909,8 +11913,6 @@ int32_t clipmove(vec3_t *pos, int16_t *sectnum,
                 goalx = mulscale20(lx,i)+intx;
                 goaly = mulscale20(ly,i)+inty;
             }
-//            else if (tempint2<0)
-//                Bprintf("!! tempint2<0 !!\n");
 
             tempint1 = dmulscale6(lx,oxvect,ly,oyvect);
             for (i=cnt+1; i<=clipmoveboxtracenum; i++)
diff --git a/polymer/eduke32/build/src/polymost.c b/polymer/eduke32/build/src/polymost.c
index d4c2129a3..7498bf122 100644
--- a/polymer/eduke32/build/src/polymost.c
+++ b/polymer/eduke32/build/src/polymost.c
@@ -4503,8 +4503,8 @@ void polymost_drawrooms()
                 int32_t scrv[2] = {(vx>>12), (vy>>12)};
                 int32_t scrv_r[2] = {scrv[1], -scrv[0]};
                 walltype *wal = &wall[sector[searchsector].wallptr];
-                int32_t wdistsq, bestwdistsq=0x7fffffff;
-                int16_t k, bestk=-1;
+                uint64_t wdistsq, bestwdistsq=0x7fffffff;
+                int32_t k, bestk=-1;
 
                 for (k=0; k<sector[searchsector].wallnum; k++)
                 {
@@ -4516,6 +4516,7 @@ void polymost_drawrooms()
                     float w1d = (float)(scrv_r[0]*pw1[0] + scrv_r[1]*pw1[1]);
                     float w2d = (float)(scrv_r[0]*pw2[0] + scrv_r[1]*pw2[1]);
                     int32_t ptonline[2], scrp[2];
+                    int64_t t1, t2;
 
                     w2d = -w2d;
                     if ((w1d==0 && w2d==0) || (w1d<0 || w2d<0))
@@ -4526,7 +4527,8 @@ void polymost_drawrooms()
                     scrp[1] = ptonline[1]-vect.y;
                     if (scrv[0]*scrp[0] + scrv[1]*scrp[1] <= 0)
                         continue;
-                    wdistsq = scrp[0]*scrp[0] + scrp[1]*scrp[1];
+                    t1=scrp[0]; t2=scrp[1];
+                    wdistsq = t1*t1 + t2*t2;
                     if (wdistsq < bestwdistsq)
                     {
                         bestk = k;