Mark portals in scene alpha channel for the SSAO pass

src/gl/renderer/gl_postprocess.cpp

@@ -170,8 +170,13 @@ void FGLRenderer::AmbientOccludeScene()
 	mBuffers->BindSceneDepthTexture(0);
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+	mBuffers->BindSceneColorTexture(1);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+	glActiveTexture(GL_TEXTURE0);
 	mLinearDepthShader->Bind(multisample);
 	mLinearDepthShader->DepthTexture[multisample].Set(0);
+	mLinearDepthShader->ColorTexture[multisample].Set(1);
 	if (multisample) mLinearDepthShader->SampleCount[multisample].Set(gl_multisample);
 	mLinearDepthShader->LinearizeDepthA[multisample].Set(1.0f / GetZFar() - 1.0f / GetZNear());
 	mLinearDepthShader->LinearizeDepthB[multisample].Set(MAX(1.0f / GetZNear(), 1.e-8f));

src/gl/scene/gl_portal.cpp

@@ -423,14 +423,16 @@ void GLPortal::End(bool usestencil)
 		glDepthFunc(GL_LEQUAL);
 		glDepthRange(0, 1);
 		{
-			ScopedColorMask colorMask(0, 0, 0, 0); 
-			// glColorMask(0,0,0,0);						// no graphics
+			ScopedColorMask colorMask(0, 0, 0, 1); // mark portal in alpha channel but don't touch color
 			gl_RenderState.SetEffect(EFF_STENCIL);
 			gl_RenderState.EnableTexture(false);
+			gl_RenderState.BlendFunc(GL_ONE, GL_ZERO);
+			gl_RenderState.BlendEquation(GL_FUNC_ADD);
+			gl_RenderState.Apply();
 			DrawPortalStencil();
 			gl_RenderState.SetEffect(EFF_NONE);
 			gl_RenderState.EnableTexture(true);
-		} // glColorMask(1, 1, 1, 1);
+		}
 		glDepthFunc(GL_LESS);
 	}
 	PortalAll.Unclock();

src/gl/shaders/gl_ambientshader.cpp

@@ -60,6 +60,7 @@ void FLinearDepthShader::Bind(bool multisample)
 		shader.Link("shaders/glsl/lineardepth");
 		shader.SetAttribLocation(0, "PositionInProjection");
 		DepthTexture[multisample].Init(shader, "DepthTexture");
+		ColorTexture[multisample].Init(shader, "ColorTexture");
 		SampleCount[multisample].Init(shader, "SampleCount");
 		LinearizeDepthA[multisample].Init(shader, "LinearizeDepthA");
 		LinearizeDepthB[multisample].Init(shader, "LinearizeDepthB");

src/gl/shaders/gl_ambientshader.h

@@ -9,6 +9,7 @@ public:
 	void Bind(bool multisample);
 
 	FBufferedUniformSampler DepthTexture[2];
+	FBufferedUniformSampler ColorTexture[2];
 	FBufferedUniform1i SampleCount[2];
 	FBufferedUniform1f LinearizeDepthA[2];
 	FBufferedUniform1f LinearizeDepthB[2];

wadsrc/static/shaders/glsl/lineardepth.fp

@@ -4,9 +4,11 @@ out vec4 FragColor;
 
 #if defined(MULTISAMPLE)
 uniform sampler2DMS DepthTexture;
+uniform sampler2DMS ColorTexture;
 uniform int SampleCount;
 #else
 uniform sampler2D DepthTexture;
+uniform sampler2D ColorTexture;
 #endif
 
 uniform float LinearizeDepthA;
@@ -25,15 +27,15 @@ void main()
 	ivec2 ipos = ivec2(uv * vec2(texSize));
 	float depth = 0.0;
 	for (int i = 0; i < SampleCount; i++)
-		depth += texelFetch(DepthTexture, ipos, i).x;
+		depth += texelFetch(ColorTexture, ipos, i).a != 0.0 ? texelFetch(DepthTexture, ipos, i).x : 1.0;
 	depth /= float(SampleCount);
 #else
 	/*ivec2 texSize = textureSize(DepthTexture, 0);
 	ivec2 ipos = ivec2(uv * vec2(texSize));
 	if (ipos.x < 0) ipos.x += texSize.x;
 	if (ipos.y < 0) ipos.y += texSize.y;
-	float depth = texelFetch(DepthTexture, ipos, 0).x;*/
-	float depth = texture(DepthTexture, uv).x;
+	float depth = texelFetch(ColorTexture, ipos, 0).a != 0.0 ? texelFetch(DepthTexture, ipos, 0).x : 1.0;*/
+	float depth = texture(ColorTexture, uv).a != 0.0 ? texture(DepthTexture, uv).x : 1.0;
 #endif
 
 	float normalizedDepth = clamp(InverseDepthRangeA * depth + InverseDepthRangeB, 0.0, 1.0);

wadsrc/static/shaders/glsl/stencil.fp

@@ -3,6 +3,6 @@ out vec4 FragColor;
 
 void main()
 {
-	FragColor = vec4(1.0);
+	FragColor = vec4(1.0, 1.0, 1.0, 0.0);
 }