From e891911a991672c6fff8008b013928648fef793c Mon Sep 17 00:00:00 2001
From: Magnus Norddahl <dpjudas@users.noreply.github.com>
Date: Sat, 22 Oct 2016 00:09:06 +0200
Subject: [PATCH] Fix broken SSAO portal rendering and let FRenderState do the
 glDrawBuffers calls

---
 src/gl/renderer/gl_renderstate.h    | 17 ++++++++++++++
 src/gl/scene/gl_scene.cpp           | 35 ++++++++---------------------
 src/gl/scene/gl_walls_draw.cpp      |  2 ++
 src/gl/stereo3d/scoped_color_mask.h |  2 ++
 4 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/src/gl/renderer/gl_renderstate.h b/src/gl/renderer/gl_renderstate.h
index 3276502f1..76e2c791c 100644
--- a/src/gl/renderer/gl_renderstate.h
+++ b/src/gl/renderer/gl_renderstate.h
@@ -120,6 +120,7 @@ class FRenderState
 	FShader *activeShader;
 
 	EPassType mPassType = NORMAL_PASS;
+	int mNumDrawBuffers = 1;
 
 	bool ApplyShader();
 
@@ -491,6 +492,22 @@ public:
 		return mPassType;
 	}
 
+	void EnableDrawBuffers(int count)
+	{
+		count = MIN(count, 3);
+		if (mNumDrawBuffers != count)
+		{
+			static GLenum buffers[] = { GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2 };
+			glDrawBuffers(count, buffers);
+			mNumDrawBuffers = count;
+		}
+	}
+
+	int GetPassDrawBufferCount()
+	{
+		return mPassType == GBUFFER_PASS ? 3 : 1;
+	}
+
 	// Backwards compatibility crap follows
 	void ApplyFixedFunction();
 	void DrawColormapOverlay();
diff --git a/src/gl/scene/gl_scene.cpp b/src/gl/scene/gl_scene.cpp
index 13eb254ce..b1876c5be 100644
--- a/src/gl/scene/gl_scene.cpp
+++ b/src/gl/scene/gl_scene.cpp
@@ -160,9 +160,8 @@ void FGLRenderer::Set3DViewport(bool mainview)
 	{
 		bool useSSAO = (gl_ssao != 0);
 		mBuffers->BindSceneFB(useSSAO);
-		GLenum buffers[] = { GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2 };
-		glDrawBuffers(useSSAO ? 3 : 1, buffers);
 		gl_RenderState.SetPassType(useSSAO ? GBUFFER_PASS : NORMAL_PASS);
+		gl_RenderState.EnableDrawBuffers(gl_RenderState.GetPassDrawBufferCount());
 		gl_RenderState.Apply();
 	}
 
@@ -479,6 +478,11 @@ void FGLRenderer::DrawScene(int drawmode)
 	static int recursion=0;
 	static int ssao_portals_available = 0;
 
+	if (drawmode == DM_MAINVIEW)
+		ssao_portals_available = gl_ssao_portals + 1;
+	else if (drawmode == DM_OFFSCREEN)
+		ssao_portals_available = 0;
+
 	if (camera != nullptr)
 	{
 		ActorRenderFlags savedflags = camera->renderflags;
@@ -495,35 +499,14 @@ void FGLRenderer::DrawScene(int drawmode)
 	}
 	GLRenderer->mClipPortal = NULL;	// this must be reset before any portal recursion takes place.
 
-	// Decide if we need to do ssao for this scene
-	bool applySSAO = gl_ssao != 0 && FGLRenderBuffers::IsEnabled();
-	switch (drawmode)
-	{
-	case DM_MAINVIEW: ssao_portals_available = gl_ssao_portals; break;
-	case DM_OFFSCREEN: ssao_portals_available = 0; applySSAO = false; break;
-	case DM_PORTAL: applySSAO = applySSAO && (ssao_portals_available > 0); ssao_portals_available--; break;
-	}
-
-	// If SSAO is active, switch to gbuffer shaders and use the framebuffer with gbuffers
-	if (applySSAO)
-	{
-		GLenum buffers[] = { GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2 };
-		glDrawBuffers(3, buffers);
-		gl_RenderState.SetPassType(GBUFFER_PASS);
-		gl_RenderState.Apply();
-		gl_RenderState.ApplyMatrices();
-	}
-
 	RenderScene(recursion);
 
-	// Apply ambient occlusion and switch back to shaders without gbuffer output
-	if (applySSAO)
+	if (ssao_portals_available > 0 && gl_RenderState.GetPassType() == GBUFFER_PASS)
 	{
-		GLenum buffers[] = { GL_COLOR_ATTACHMENT0 };
-		glDrawBuffers(1, buffers);
+		gl_RenderState.EnableDrawBuffers(1);
 		AmbientOccludeScene();
 		mBuffers->BindSceneFB(true);
-		gl_RenderState.SetPassType(NORMAL_PASS);
+		gl_RenderState.EnableDrawBuffers(gl_RenderState.GetPassDrawBufferCount());
 		gl_RenderState.Apply();
 		gl_RenderState.ApplyMatrices();
 	}
diff --git a/src/gl/scene/gl_walls_draw.cpp b/src/gl/scene/gl_walls_draw.cpp
index dbc43fd22..480ce4e86 100644
--- a/src/gl/scene/gl_walls_draw.cpp
+++ b/src/gl/scene/gl_walls_draw.cpp
@@ -227,6 +227,7 @@ void GLWall::RenderFogBoundary()
 		{
 			int rel = rellight + getExtraLight();
 			gl_SetFog(lightlevel, rel, &Colormap, false);
+			gl_RenderState.EnableDrawBuffers(1);
 			gl_RenderState.SetEffect(EFF_FOGBOUNDARY);
 			gl_RenderState.AlphaFunc(GL_GEQUAL, 0.f);
 			glEnable(GL_POLYGON_OFFSET_FILL);
@@ -235,6 +236,7 @@ void GLWall::RenderFogBoundary()
 			glPolygonOffset(0.0f, 0.0f);
 			glDisable(GL_POLYGON_OFFSET_FILL);
 			gl_RenderState.SetEffect(EFF_NONE);
+			gl_RenderState.EnableDrawBuffers(gl_RenderState.GetPassDrawBufferCount());
 		}
 		else
 		{
diff --git a/src/gl/stereo3d/scoped_color_mask.h b/src/gl/stereo3d/scoped_color_mask.h
index 42c02db3a..e55772092 100644
--- a/src/gl/stereo3d/scoped_color_mask.h
+++ b/src/gl/stereo3d/scoped_color_mask.h
@@ -41,8 +41,10 @@ public:
 		gl_RenderState.GetColorMask(saved[0], saved[1], saved[2], saved[3]);
 		gl_RenderState.SetColorMask(r, g, b, a);
 		gl_RenderState.ApplyColorMask();
+		gl_RenderState.EnableDrawBuffers(1);
 	}
 	~ScopedColorMask() {
+		gl_RenderState.EnableDrawBuffers(gl_RenderState.GetPassDrawBufferCount());
 		gl_RenderState.SetColorMask(saved[0], saved[1], saved[2], saved[3]);
 		gl_RenderState.ApplyColorMask();
 	}