From 7d21fe6d754c937c0c5c1483043dd83ee263ab1f Mon Sep 17 00:00:00 2001
From: Magnus Norddahl <dpjudas@users.noreply.github.com>
Date: Sat, 29 Jul 2017 19:26:53 +0200
Subject: [PATCH] - Fix mouse position being wrong when vid_scalemode is active

---
 src/gl/system/gl_framebuffer.cpp   | 15 ++++++++
 src/gl/system/gl_framebuffer.h     |  2 ++
 src/gl/system/gl_swframebuffer.cpp | 55 +++++++++++++++++++++---------
 src/gl/system/gl_swframebuffer.h   |  4 +++
 4 files changed, 59 insertions(+), 17 deletions(-)

diff --git a/src/gl/system/gl_framebuffer.cpp b/src/gl/system/gl_framebuffer.cpp
index 73b81bd2c..f4e68791e 100644
--- a/src/gl/system/gl_framebuffer.cpp
+++ b/src/gl/system/gl_framebuffer.cpp
@@ -548,3 +548,18 @@ void OpenGLFrameBuffer::GameRestart()
 	gl_GenerateGlobalBrightmapFromColormap();
 }
 
+
+void OpenGLFrameBuffer::ScaleCoordsFromWindow(int16_t &x, int16_t &y)
+{
+	int letterboxX = GLRenderer->mOutputLetterbox.left;
+	int letterboxY = GLRenderer->mOutputLetterbox.top;
+	int letterboxWidth = GLRenderer->mOutputLetterbox.width;
+	int letterboxHeight = GLRenderer->mOutputLetterbox.height;
+
+	// Subtract the LB video mode letterboxing
+	if (IsFullscreen())
+		y -= (GetTrueHeight() - VideoHeight) / 2;
+
+	x = int16_t((x - letterboxX) * Width / letterboxWidth);
+	y = int16_t((y - letterboxY) * Height / letterboxHeight);
+}
diff --git a/src/gl/system/gl_framebuffer.h b/src/gl/system/gl_framebuffer.h
index 0d88667ca..7d11db921 100644
--- a/src/gl/system/gl_framebuffer.h
+++ b/src/gl/system/gl_framebuffer.h
@@ -82,6 +82,8 @@ public:
 
 	void SetVSync(bool vsync);
 
+	void ScaleCoordsFromWindow(int16_t &x, int16_t &y) override;
+
 	bool HWGammaActive = false;			// Are we using hardware or software gamma?
 	std::shared_ptr<FGLDebug> mDebug;	// Debug API
 private:
diff --git a/src/gl/system/gl_swframebuffer.cpp b/src/gl/system/gl_swframebuffer.cpp
index dd552f6df..199dd2826 100644
--- a/src/gl/system/gl_swframebuffer.cpp
+++ b/src/gl/system/gl_swframebuffer.cpp
@@ -714,6 +714,29 @@ void OpenGLSWFrameBuffer::DrawTriangleList(int minIndex, int numVertices, int st
 	glDrawRangeElements(GL_TRIANGLES, minIndex, minIndex + numVertices - 1, primitiveCount * 3, GL_UNSIGNED_SHORT, (const void*)(startIndex * sizeof(uint16_t)));
 }
 
+void OpenGLSWFrameBuffer::GetLetterboxFrame(int &letterboxX, int &letterboxY, int &letterboxWidth, int &letterboxHeight)
+{
+	int clientWidth = GetClientWidth();
+	int clientHeight = GetClientHeight();
+
+	float scaleX, scaleY;
+	if (ViewportIsScaled43())
+	{
+		scaleX = MIN(clientWidth / (float)Width, clientHeight / (Height * 1.2f));
+		scaleY = scaleX * 1.2f;
+	}
+	else
+	{
+		scaleX = MIN(clientWidth / (float)Width, clientHeight / (float)Height);
+		scaleY = scaleX;
+	}
+
+	letterboxWidth = (int)round(Width * scaleX);
+	letterboxHeight = (int)round(Height * scaleY);
+	letterboxX = (clientWidth - letterboxWidth) / 2;
+	letterboxY = (clientHeight - letterboxHeight) / 2;
+}
+
 void OpenGLSWFrameBuffer::Present()
 {
 	int clientWidth = GetClientWidth();
@@ -723,23 +746,8 @@ void OpenGLSWFrameBuffer::Present()
 		glBindFramebuffer(GL_FRAMEBUFFER, 0);
 		glViewport(0, 0, clientWidth, clientHeight);
 
-		float scaleX, scaleY;
-		if (ViewportIsScaled43())
-		{
-			scaleX = MIN(clientWidth / (float)Width, clientHeight / (Height * 1.2f));
-			scaleY = scaleX * 1.2f;
-		}
-		else
-		{
-			scaleX = MIN(clientWidth / (float)Width, clientHeight / (float)Height);
-			scaleY = scaleX;
-		}
-
-		int letterboxWidth = (int)round(Width * scaleX);
-		int letterboxHeight = (int)round(Height * scaleY);
-		int letterboxX = (clientWidth - letterboxWidth) / 2;
-		int letterboxY = (clientHeight - letterboxHeight) / 2;
-
+		int letterboxX, letterboxY, letterboxWidth, letterboxHeight;
+		GetLetterboxFrame(letterboxX, letterboxY, letterboxWidth, letterboxHeight);
 		DrawLetterbox(letterboxX, letterboxY, letterboxWidth, letterboxHeight);
 		glViewport(letterboxX, letterboxY, letterboxWidth, letterboxHeight);
 
@@ -3830,3 +3838,16 @@ void OpenGLSWFrameBuffer::SetPaletteTexture(HWTexture *texture, int count, uint3
 	SetConstant(PSCONST_PaletteMod, 255 * fcount, 0.5f * fcount, 0, 0);
 	SetTexture(1, texture);
 }
+
+void OpenGLSWFrameBuffer::ScaleCoordsFromWindow(int16_t &x, int16_t &y)
+{
+	int letterboxX, letterboxY, letterboxWidth, letterboxHeight;
+	GetLetterboxFrame(letterboxX, letterboxY, letterboxWidth, letterboxHeight);
+
+	// Subtract the LB video mode letterboxing
+	if (IsFullscreen())
+		y -= (GetTrueHeight() - VideoHeight) / 2;
+
+	x = int16_t((x - letterboxX) * Width / letterboxWidth);
+	y = int16_t((y - letterboxY) * Height / letterboxHeight);
+}
diff --git a/src/gl/system/gl_swframebuffer.h b/src/gl/system/gl_swframebuffer.h
index 93f7c905c..e29ff40ba 100644
--- a/src/gl/system/gl_swframebuffer.h
+++ b/src/gl/system/gl_swframebuffer.h
@@ -72,6 +72,8 @@ public:
 	int GetTrueHeight() override { return TrueHeight; }
 #endif
 
+	void ScaleCoordsFromWindow(int16_t &x, int16_t &y) override;
+
 private:
 	struct FBVERTEX
 	{
@@ -194,6 +196,8 @@ private:
 	void DrawLineList(int count);
 	void DrawTriangleList(int minIndex, int numVertices, int startIndex, int primitiveCount);
 	void Present();
+
+	void GetLetterboxFrame(int &x, int &y, int &width, int &height);
 	
 	static void BgraToRgba(uint32_t *dest, const uint32_t *src, int width, int height, int srcpitch);