From 2e2b110ecbd784e369901dfa6c9f7bd54b2c0572 Mon Sep 17 00:00:00 2001
From: Eric Wasylishen <ewasylishen@gmail.com>
Date: Mon, 18 May 2015 06:11:56 +0000
Subject: [PATCH] r_shadows: Better stencil shadow rendering from MarkV.

Now an 8-bit stencil buffer is requested when using a 24-bit depth buffer (the default).

git-svn-id: svn://svn.code.sf.net/p/quakespasm/code/trunk/quakespasm@1209 af15c1b1-3010-417e-b628-4374ebc0bcbd
---
 Quake/gl_rmain.c  | 14 ++++++++++++++
 Quake/gl_vidsdl.c | 20 ++++++++++++++++++--
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/Quake/gl_rmain.c b/Quake/gl_rmain.c
index d05ad286..3c029958 100644
--- a/Quake/gl_rmain.c
+++ b/Quake/gl_rmain.c
@@ -852,6 +852,15 @@ void R_DrawShadows (void)
 	if (!r_shadows.value || !r_drawentities.value || r_drawflat_cheatsafe || r_lightmap_cheatsafe)
 		return;
 
+	// Use stencil buffer to prevent self-intersecting shadows, from Baker (MarkV)
+	if (gl_stencilbits)
+	{
+		glClear(GL_STENCIL_BUFFER_BIT);
+		glStencilFunc(GL_EQUAL, 0, ~0);
+		glStencilOp(GL_KEEP, GL_KEEP, GL_INCR);
+		glEnable(GL_STENCIL_TEST);
+	}
+
 	for (i=0 ; i<cl_numvisedicts ; i++)
 	{
 		currententity = cl_visedicts[i];
@@ -864,6 +873,11 @@ void R_DrawShadows (void)
 
 		GL_DrawAliasShadow (currententity);
 	}
+
+	if (gl_stencilbits)
+	{
+		glDisable(GL_STENCIL_TEST);
+	}
 }
 
 /*
diff --git a/Quake/gl_vidsdl.c b/Quake/gl_vidsdl.c
index a62efab7..32f5a216 100644
--- a/Quake/gl_vidsdl.c
+++ b/Quake/gl_vidsdl.c
@@ -536,7 +536,7 @@ static qboolean VID_SetMode (int width, int height, int bpp, qboolean fullscreen
 	int		temp;
 	Uint32	flags;
 	char		caption[50];
-	int		depthbits;
+	int		depthbits, stencilbits;
 	int		fsaa_obtained;
 	
 	// so Con_Printfs don't mess us up by forcing vid and snd updates
@@ -548,9 +548,17 @@ static qboolean VID_SetMode (int width, int height, int bpp, qboolean fullscreen
 
 	/* z-buffer depth */
 	if (bpp == 16)
+	{
 		depthbits = 16;
-	else	depthbits = 24;
+		stencilbits = 0;
+	}
+	else
+	{
+		depthbits = 24;
+		stencilbits = 8;
+	}
 	SDL_GL_SetAttribute(SDL_GL_DEPTH_SIZE, depthbits);
+	SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, stencilbits);
 
 	/* fsaa */
 	SDL_GL_SetAttribute(SDL_GL_MULTISAMPLEBUFFERS, fsaa > 0 ? 1 : 0);
@@ -574,6 +582,10 @@ static qboolean VID_SetMode (int width, int height, int bpp, qboolean fullscreen
 			SDL_GL_SetAttribute(SDL_GL_DEPTH_SIZE, 16);
 			draw_context = SDL_CreateWindow (caption, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, width, height, flags);
 		}
+		if (!draw_context) { // scale back SDL_GL_STENCIL_SIZE
+			SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 0);
+			draw_context = SDL_CreateWindow (caption, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, width, height, flags);
+		}
 		if (!draw_context)
 			Sys_Error ("Couldn't create window");
 
@@ -631,6 +643,10 @@ static qboolean VID_SetMode (int width, int height, int bpp, qboolean fullscreen
 	if (!draw_context) { // scale back SDL_GL_DEPTH_SIZE
 		SDL_GL_SetAttribute(SDL_GL_DEPTH_SIZE, 16);
 		draw_context = SDL_SetVideoMode(width, height, bpp, flags);
+	}
+	if (!draw_context) { // scale back SDL_GL_STENCIL_SIZE
+		SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 0);
+		draw_context = SDL_SetVideoMode(width, height, bpp, flags);
 		if (!draw_context)
 			Sys_Error ("Couldn't set video mode");
 	}