Use OpenGL2 glStencilOpSeparate() for shadows, if available

can be enabled/disabled with the r_useStencilOpSeparate for comparisons

(like Z-Fail, this doesn't really seem to make a difference on my main
 machine, neither on my RPi4)

Partly based on Pat Raynor's Code:
2933cb5545/neo/renderer/draw_stencilshadow.cpp

neo/renderer/RenderSystem_init.cpp

@@ -227,7 +227,7 @@ idCVar r_debugRenderToTexture( "r_debugRenderToTexture", "0", CVAR_RENDERER | CV
 idCVar r_scaleMenusTo43( "r_scaleMenusTo43", "1", CVAR_RENDERER | CVAR_ARCHIVE | CVAR_BOOL, "Scale menus, fullscreen videos and PDA to 4:3 aspect ratio" );
 // DG: the fscking patent has finally expired
 idCVar r_useCarmacksReverse( "r_useCarmacksReverse", "1", CVAR_RENDERER | CVAR_ARCHIVE | CVAR_BOOL, "Use Z-Fail (Carmack's Reverse) when rendering shadows" );
-
+idCVar r_useStencilOpSeparate( "r_useStencilOpSeparate", "1", CVAR_RENDERER | CVAR_ARCHIVE | CVAR_BOOL, "Use glStencilOpSeparate() (if available) when rendering shadows" );
 
 // define qgl functions
 #define QGLPROC(name, rettype, args) rettype (APIENTRYP q##name) args;
@@ -275,6 +275,9 @@ PFNGLPROGRAMLOCALPARAMETER4FVARBPROC	qglProgramLocalParameter4fvARB;
 // GL_EXT_depth_bounds_test
 PFNGLDEPTHBOUNDSEXTPROC                 qglDepthBoundsEXT;
 
+// DG: couldn't find any extension for this, it's supported in GL2.0 and newer, incl OpenGL ES2.0
+PFNGLSTENCILOPSEPARATEPROC qglStencilOpSeparate;
+
 /*
 =================
 R_CheckExtension
@@ -393,6 +396,15 @@ static void R_CheckPortableExtensions( void ) {
 	if ( glConfig.twoSidedStencilAvailable )
 		qglActiveStencilFaceEXT = (PFNGLACTIVESTENCILFACEEXTPROC)GLimp_ExtensionPointer( "glActiveStencilFaceEXT" );
 
+	if( glConfig.glVersion >= 2.0) {
+		common->Printf( "... got GL2.0+ glStencilOpSeparate()\n" );
+		qglStencilOpSeparate = (PFNGLSTENCILOPSEPARATEPROC)GLimp_ExtensionPointer( "glStencilOpSeparate" );
+	} else {
+		// TODO: there was an extension by ATI providing glStencilOpSeparateATI - do we care?
+		common->Printf( "... don't have GL2.0+ glStencilOpSeparate()\n" );
+		qglStencilOpSeparate = nullptr;
+	}
+
 	// ARB_vertex_buffer_object
 	glConfig.ARBVertexBufferObjectAvailable = R_CheckExtension( "GL_ARB_vertex_buffer_object" );
 	if(glConfig.ARBVertexBufferObjectAvailable) {

neo/renderer/draw_common.cpp

@@ -32,7 +32,7 @@ If you have questions concerning this license or the applicable additional terms
 #include "renderer/tr_local.h"
 
 extern idCVar r_useCarmacksReverse;
-
+extern idCVar r_useStencilOpSeparate;
 /*
 =====================
 RB_BakeTextureMatrixIntoTexgen
@@ -1142,8 +1142,27 @@ static void RB_T_Shadow( const drawSurf_t *surf ) {
 		return;
 	}
 
-	if( !r_useCarmacksReverse.GetBool() ) { // LEITH: the original patent free "preload" code
+	// DG: that bloody patent on depth-fail stencil shadows has finally expired on 2019-10-13,
+	//     so use them (see https://patents.google.com/patent/US6384822B1/en for expiration status)
+	bool useStencilOpSeperate = r_useStencilOpSeparate.GetBool() && qglStencilOpSeparate != nullptr;
+	if( !r_useCarmacksReverse.GetBool() ) {
+		if( useStencilOpSeperate ) {
+			// not using z-fail, but using qglStencilOpSeparate()
+			GLenum firstFace = backEnd.viewDef->isMirror ? GL_FRONT : GL_BACK;
+			GLenum secondFace = backEnd.viewDef->isMirror ? GL_BACK : GL_FRONT;
+			GL_Cull( CT_TWO_SIDED );
+			if ( !external ) {
+				qglStencilOpSeparate( firstFace, GL_KEEP, tr.stencilDecr, tr.stencilDecr );
+				qglStencilOpSeparate( secondFace, GL_KEEP, tr.stencilIncr, tr.stencilIncr );
+				RB_DrawShadowElementsWithCounters( tri, numIndexes );
+			}
 
+			qglStencilOpSeparate( firstFace, GL_KEEP, GL_KEEP, tr.stencilIncr );
+			qglStencilOpSeparate( secondFace, GL_KEEP, GL_KEEP, tr.stencilDecr );
+
+			RB_DrawShadowElementsWithCounters( tri, numIndexes );
+
+		} else { // DG: this is the original code:
 			// patent-free work around
 			if ( !external ) {
 				// "preload" the stencil buffer with the number of volumes
@@ -1164,11 +1183,25 @@ static void RB_T_Shadow( const drawSurf_t *surf ) {
 			qglStencilOp( GL_KEEP, GL_KEEP, tr.stencilDecr );
 			GL_Cull( CT_BACK_SIDED );
 			RB_DrawShadowElementsWithCounters( tri, numIndexes );
+		}
+	} else { // use the formerly patented "Carmack's Reverse" Z-Fail code
+		if( useStencilOpSeperate ) {
+			// Z-Fail with glStencilOpSeparate() which will reduce draw calls
+			GLenum firstFace = backEnd.viewDef->isMirror ? GL_FRONT : GL_BACK;
+			GLenum secondFace = backEnd.viewDef->isMirror ? GL_BACK : GL_FRONT;
+			if ( !external ) { // z-fail
+				qglStencilOpSeparate( firstFace, GL_KEEP, tr.stencilDecr, GL_KEEP );
+				qglStencilOpSeparate( secondFace, GL_KEEP, tr.stencilIncr, GL_KEEP );
+			} else { // depth-pass
+				qglStencilOpSeparate( firstFace, GL_KEEP, GL_KEEP, tr.stencilIncr );
+				qglStencilOpSeparate( secondFace, GL_KEEP, GL_KEEP, tr.stencilDecr );
+			}
+			GL_Cull( CT_TWO_SIDED );
+			RB_DrawShadowElementsWithCounters( tri, numIndexes );
 
-	} else { // LEITH: the (formerly patented) "Carmack's Reverse" code
+		} else { // Z-Fail without glStencilOpSeparate()
 
-		// DG: that bloody patent on depth-fail stencil shadows has finally expired on 2019-10-13,
-		//     so use them (see https://patents.google.com/patent/US6384822B1/en for expiration status)
+			// LEITH: the (formerly patented) "Carmack's Reverse" code
 
 			// depth-fail/Z-Fail stencil shadows
 			if ( !external ) {
@@ -1190,7 +1223,7 @@ static void RB_T_Shadow( const drawSurf_t *surf ) {
 				RB_DrawShadowElementsWithCounters( tri, numIndexes );
 			}
 		}
-
+	}
 }
 
 /*

neo/renderer/qgl.h

@@ -95,6 +95,9 @@ extern	void ( APIENTRY *qglColorTableEXT)( int, int, int, int, int, const void *
 // EXT_stencil_two_side
 extern	PFNGLACTIVESTENCILFACEEXTPROC	qglActiveStencilFaceEXT;
 
+// DG: couldn't find any extension for this, it's supported in GL2.0 and newer, incl OpenGL ES2.0
+extern PFNGLSTENCILOPSEPARATEPROC qglStencilOpSeparate;
+
 // ARB_texture_compression
 extern	PFNGLCOMPRESSEDTEXIMAGE2DARBPROC	qglCompressedTexImage2DARB;
 extern	PFNGLGETCOMPRESSEDTEXIMAGEARBPROC	qglGetCompressedTexImageARB;