Increase stack size on Windows to 8MB, so huge models work

The HD version of the "Birdman" Hellknight model crashed the game,
because on Linux it triggered the 600KB _alloca() assertion and on
Windows (that didn't have such an assertion up to this commit) it
overflowed the stack.
Thde default stacksize on Windows is 1MB, increasing that to 8MB fixes the
problem (in Doom3 1.3.1 it was 4MB).
I also implemented the _alloca() size check assertion for Windows, and
increased the allowed size to 2MB (so it can be safely called multiple
times per function)
Model: https://www.moddb.com/mods/birdman-doom3/downloads/birdman-doom3-v11
This commit is contained in:
Daniel Gibson 2023-01-21 19:26:11 +01:00
parent e7e06d05f1
commit dee76dbb21
2 changed files with 23 additions and 7 deletions

View file

@ -364,7 +364,7 @@ if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID STREQUAL "Clang")
set(sys_libs ${sys_libs} "-framework Carbon -framework Cocoa -framework IOKit")
elseif(WIN32)
set(ldflags "${ldflags} -static-libgcc -static-libstdc++")
set(ldflags "${ldflags} -static-libgcc -static-libstdc++ -Wl,--stack,8388608")
elseif(os STREQUAL "linux")
set(sys_libs ${sys_libs} dl)
elseif(os STREQUAL "freebsd")
@ -393,6 +393,8 @@ elseif(MSVC)
set(CMAKE_C_FLAGS_RELEASE "/Ox /Oy /MD")
set(CMAKE_C_FLAGS_RELWITHDEBINFO "/Ox /Oy /Zi /MD")
set(CMAKE_C_FLAGS_MINSIZEREL "/Ox /Oy /Os /MD")
# use 8MB stack instead of 1MB, so big models don't overflow it with _alloca()
set(ldflags "${ldflags} /STACK:8388608")
else()
message(FATAL_ERROR "Unsupported compiler")
endif()

View file

@ -32,6 +32,16 @@ If you have questions concerning this license or the applicable additional terms
#include "config.h"
#include "framework/BuildDefines.h"
#ifdef _WIN32
#include <malloc.h> // _alloca()
#endif
// NOTE: By default Win32 uses a 1MB stack. Doom3 1.3.1 uses 4MB (probably set after compiling with EDITBIN /STACK
// dhewm3 now uses a 8MB stack, set with a linker flag in CMakeLists.txt (/STACK:8388608 for MSVC, -Wl,--stack,8388608 for mingw)
// Linux has a 8MB stack by default, and so does macOS, at least for the main thread
// anyway, a 2MB limit alloca should be safe even when using it multiple times in the same function
#define ID_MAX_ALLOCA_SIZE 2097152 // 2MB
/*
===============================================================================
@ -71,7 +81,14 @@ If you have questions concerning this license or the applicable additional terms
// Win32
#if defined(WIN32) || defined(_WIN32)
#define _alloca16( x ) ((void *)((((uintptr_t)_alloca( (x)+15 )) + 15) & ~15))
#ifdef __MINGW32__
#undef _alloca // in mingw _alloca is a #define
#define _alloca16( x ) ( (assert((x)<ID_MAX_ALLOCA_SIZE)), __builtin_alloca_with_align( (x), 16*8 ) )
#define _alloca( x ) ( (assert((x)<ID_MAX_ALLOCA_SIZE)), __builtin_alloca( (x) ) )
#else
#define _alloca16( x ) ( (void *) ( (assert((x)<ID_MAX_ALLOCA_SIZE)), ((((uintptr_t)_alloca( (x)+15 )) + 15) & ~15) ) )
#define _alloca( x ) ( (void *) ( (assert((x)<ID_MAX_ALLOCA_SIZE)), _alloca( (x) ) ) )
#endif
#define PATHSEPERATOR_STR "\\"
#define PATHSEPERATOR_CHAR '\\'
@ -168,8 +185,8 @@ If you have questions concerning this license or the applicable additional terms
// Unix
#ifdef __unix__
#define _alloca( x ) (({assert( (x)<600000 );}), alloca( (x) ))
#define _alloca16( x ) (({assert( (x)<600000 );}),((void *)((((uintptr_t)alloca( (x)+15 )) + 15) & ~15)))
#define _alloca( x ) (({assert( (x)<ID_MAX_ALLOCA_SIZE );}), alloca( (x) ))
#define _alloca16( x ) (({assert( (x)<ID_MAX_ALLOCA_SIZE );}),((void *)((((uintptr_t)alloca( (x)+15 )) + 15) & ~15)))
#ifdef GAME_DLL
#define ID_GAME_API __attribute__((visibility ("default")))
@ -208,9 +225,6 @@ If you have questions concerning this license or the applicable additional terms
// MSVC does not provide this C99 header
#include <inttypes.h>
#endif
#if defined(__MINGW32__)
#include <malloc.h>
#endif
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>