dhewm/dhewm3
0
0
Fork 0
mirror of https://github.com/dhewm/dhewm3.git synced 2024-11-23 04:51:56 +00:00
Code Issues Projects Releases Packages Wiki Activity

Prevent stackoverflow in R_DeriveTangentsWithoutNormals()

Browse source 
turns out tri->numIndexes can be quite big so _alloca16() would be
called with >1MB - and the Win32 stack only is 1MB, so that overflows.
As a workaround, use Mem_Alloc16() if we need >600KB

was reported in #265
This commit is contained in:
Daniel Gibson 2020-01-11 15:39:34 +01:00
parent dd7a6b2832
commit bcf647cf71
1 changed files with 12 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
neo/renderer/tr_trisurf.cpp
View file

@ -1378,7 +1378,15 @@ void R_DeriveTangentsWithoutNormals( srfTriangles_t *tri ) {
faceTangents_t *ft;
idDrawVert *vert;
faceTangents = (faceTangents_t *)_alloca16( sizeof(faceTangents[0]) * tri->numIndexes/3 );
// DG: windows only has a 1MB stack and it could happen that we try to allocate >1MB here
// (in lost mission mod, game/le_hell map), causing a stack overflow
// to prevent that, use heap allocation if it's >600KB
size_t allocaSize = sizeof(faceTangents[0]) * tri->numIndexes/3;
if(allocaSize < 600000)
faceTangents = (faceTangents_t *)_alloca16( allocaSize );
else
faceTangents = (faceTangents_t *)Mem_Alloc16( allocaSize );
R_DeriveFaceTangents( tri, faceTangents );
// clear the tangents
@ -1434,6 +1442,9 @@ void R_DeriveTangentsWithoutNormals( srfTriangles_t *tri ) {
}
tri->tangentsCalculated = true;
if(allocaSize >= 600000)
Mem_Free16( faceTangents );
}
static ID_INLINE void VectorNormalizeFast2( const idVec3 &v, idVec3 &out) {