fixed crashes due to render command list overflows

also increased the buffer's size (e.g. to draw all chars in the console in 4K)

one of the crashes happens in R_SortDrawSurfs:
-> render command list is too full
-> RE_EndFrame returns early because it can't allocate RC_SWAP_BUFFERS
-> R_ClearFrame in RE_EndFrame doesn't get called
-> the next frame starts with r_firstSceneDrawSurf etc. not being reset to 0
-> r_firstSceneDrawSurf becomes really close to the maximum draw surface limit
-> the draw surface list is iterated incorrectly (no wrapping handled)
-> we fetch a draw surface we shouldn't
-> its sort key gets decoded and we get an invalid sorted shader index
-> we fetch a NULL shader at that index location
-> we attempt to read shader->sort
-> we crash reading address 76
-> 76 bytes is exactly the offset of the sort member into the shader_t struct

changelog.txt

@@ -159,6 +159,8 @@ chg: r_fullbright is now latched again
 
 chg: negative r_swapInterval values will request adaptive V-Sync when using an OpenGL back-end
 
+fix: the engine could crash due to render command list overflows
+
 fix: made more keys bindable on Linux/FreeBSD and /bind now accepts uppercase hex digits
 
 fix: removing CVars through unset/cvar_trim/cvar_restart could leave data in an invalid state

code/renderer/tr_cmds.cpp

@@ -105,17 +105,22 @@ make sure there is enough command space, waiting on the
 render thread if needed.
 ============
 */
-void *R_GetCommandBuffer( int bytes ) {
-	renderCommandList_t	*cmdList;
-
-	cmdList = &backEndData->commands;
+void *R_GetCommandBuffer( int bytes, qbool endFrame ) {
+	const int reservedBytes = (int)( sizeof(swapBuffersCommand_t) + sizeof(screenshotCommand_t) );
+	const int endOffset = endFrame ? 0 : reservedBytes;
+	renderCommandList_t* const cmdList = &backEndData->commands;
 	bytes = PAD(bytes, sizeof(void *));
 
 	// always leave room for the end of list command
-	if ( cmdList->used + bytes + 4 > MAX_RENDER_COMMANDS ) {
-		if ( bytes > MAX_RENDER_COMMANDS - 4 ) {
+	if ( cmdList->used + bytes + endOffset > MAX_RENDER_COMMANDS ) {
+		if ( bytes > MAX_RENDER_COMMANDS - endOffset ) {
 			ri.Error( ERR_FATAL, "R_GetCommandBuffer: bad size %i", bytes );
 		}
+		if ( endFrame ) {
+			// we reserved memory specifically for this case
+			// so this really shouldn't ever happen
+			ri.Error( ERR_FATAL, "R_GetCommandBuffer: can't allocate %i bytes to end the frame", bytes );
+		}
 		// if we run out of room, just start dropping commands
 		return NULL;
 	}
@@ -128,12 +133,14 @@ void *R_GetCommandBuffer( int bytes ) {
 
 // technically, all commands should probably check tr.registered
 // but realistically, only begin+end frame really need to
-#define R_CMD(T, ID) T* cmd = (T*)R_GetCommandBuffer( sizeof(T) ); if (!cmd) return; cmd->commandId = ID;
+#define R_CMD_RET(T, ID)   T* cmd = (T*)R_GetCommandBuffer( sizeof(T), qfalse ); if (!cmd) return; cmd->commandId = ID
+#define R_CMD_NORET(T, ID) T* cmd = (T*)R_GetCommandBuffer( sizeof(T), qfalse ); if (cmd)  cmd->commandId = ID
+#define R_CMD_END(T, ID)   T* cmd = (T*)R_GetCommandBuffer( sizeof(T), qtrue  );           cmd->commandId = ID
 
 
 void R_AddDrawSurfCmd( drawSurf_t* drawSurfs, int numDrawSurfs, int numTranspSurfs )
 {
-	R_CMD( drawSurfsCommand_t, RC_DRAW_SURFS );
+	R_CMD_RET( drawSurfsCommand_t, RC_DRAW_SURFS );
 
 	cmd->drawSurfs = drawSurfs;
 	cmd->numDrawSurfs = numDrawSurfs;
@@ -148,7 +155,7 @@ void R_AddDrawSurfCmd( drawSurf_t* drawSurfs, int numDrawSurfs, int numTranspSur
 
 void RE_SetColor( const float* rgba )
 {
-	R_CMD( setColorCommand_t, RC_SET_COLOR );
+	R_CMD_RET( setColorCommand_t, RC_SET_COLOR );
 
 	if ( !rgba )
 		rgba = colorWhite;
@@ -162,7 +169,7 @@ void RE_SetColor( const float* rgba )
 
 void RE_StretchPic( float x, float y, float w, float h, float s1, float t1, float s2, float t2, qhandle_t hShader )
 {
-	R_CMD( stretchPicCommand_t, RC_STRETCH_PIC );
+	R_CMD_RET( stretchPicCommand_t, RC_STRETCH_PIC );
 
 	cmd->shader = R_GetShaderByHandle( hShader );
 	cmd->x = x;
@@ -178,7 +185,7 @@ void RE_StretchPic( float x, float y, float w, float h, float s1, float t1, floa
 
 void RE_DrawTriangle( float x0, float y0, float x1, float y1, float x2, float y2, float s0, float t0, float s1, float t1, float s2, float t2, qhandle_t hShader )
 {
-	R_CMD( triangleCommand_t, RC_TRIANGLE );
+	R_CMD_RET( triangleCommand_t, RC_TRIANGLE );
 
 	cmd->shader = R_GetShaderByHandle( hShader );
 	cmd->x0 = x0;
@@ -210,7 +217,7 @@ void RE_BeginFrame( stereoFrame_t stereoFrame )
 	if ( r_delayedScreenshotPending ) {
 		r_delayedScreenshotFrame++;
 		if ( r_delayedScreenshotFrame >= 2 ) {
-			R_CMD( screenshotCommand_t, RC_SCREENSHOT );
+			R_CMD_NORET( screenshotCommand_t, RC_SCREENSHOT );
 			*cmd = r_delayedScreenshot;
 			r_delayedScreenshotPending = qfalse;
 			r_delayedScreenshotFrame = 0;
@@ -220,7 +227,7 @@ void RE_BeginFrame( stereoFrame_t stereoFrame )
 	//
 	// draw buffer stuff
 	//
-	R_CMD( beginFrameCommand_t, RC_BEGIN_FRAME );
+	R_CMD_NORET( beginFrameCommand_t, RC_BEGIN_FRAME );
 }
 
 
@@ -249,9 +256,9 @@ void RE_EndFrame( int* pcFE, int* pc2D, int* pc3D, qbool render )
 	}
 
 	if ( render ) {
-		R_CMD( swapBuffersCommand_t, RC_SWAP_BUFFERS );
+		R_CMD_END( swapBuffersCommand_t, RC_SWAP_BUFFERS );
 		if ( delayScreenshot ) {
-			R_CMD( screenshotCommand_t, RC_SCREENSHOT );
+			R_CMD_END( screenshotCommand_t, RC_SCREENSHOT );
 			*cmd = r_delayedScreenshot;
 		}
 	} else {
@@ -279,7 +286,7 @@ void RE_EndFrame( int* pcFE, int* pc2D, int* pc3D, qbool render )
 
 void RE_TakeVideoFrame( int width, int height, byte *captureBuffer, byte *encodeBuffer, qbool motionJpeg )
 {
-	R_CMD( videoFrameCommand_t, RC_VIDEOFRAME );
+	R_CMD_RET( videoFrameCommand_t, RC_VIDEOFRAME );
 
 	cmd->width = width;
 	cmd->height = height;

code/renderer/tr_init.cpp

@@ -232,7 +232,7 @@ static void R_TakeScreenshot( const char* ext, screenshotCommand_t::ss_type type
 	} else {
 		if ( R_FindRenderCommand( RC_SCREENSHOT ) )
 			return;
-		cmd = (screenshotCommand_t*)R_GetCommandBuffer( sizeof(screenshotCommand_t) );
+		cmd = (screenshotCommand_t*)R_GetCommandBuffer( sizeof(screenshotCommand_t), qfalse );
 		if ( !cmd )
 			return;
 		cmd->delayed = qfalse;

code/renderer/tr_local.h

@@ -1398,7 +1398,7 @@ RENDERER BACK END COMMAND QUEUE
 =============================================================
 */
 
-#define	MAX_RENDER_COMMANDS	0x40000
+#define	MAX_RENDER_COMMANDS	0x80000
 
 typedef struct {
 	byte	cmds[MAX_RENDER_COMMANDS];
@@ -1599,7 +1599,7 @@ void R_BuildCloudData();
 
 void R_IssueRenderCommands();
 void* R_FindRenderCommand( renderCommand_t type );
-void *R_GetCommandBuffer( int bytes );
+void *R_GetCommandBuffer( int bytes, qbool endFrame );
 
 void R_AddDrawSurfCmd(drawSurf_t* drawSurfs, int numDrawSurfs, int numTranspSurfs );