Fix aspect ratio and texture clipping in lens shader

src/gl/renderer/gl_postprocess.cpp

@@ -98,9 +98,9 @@ CUSTOM_CVAR(Int, gl_bloom_kernel_size, 7, 0)
 
 CVAR(Bool, gl_lens, false, CVAR_ARCHIVE | CVAR_GLOBALCONFIG)
 
-CVAR(Float, gl_lens_k, -0.15f, 0)
+CVAR(Float, gl_lens_k, -0.12f, 0)
-CVAR(Float, gl_lens_kcube, 0.15f, 0)
+CVAR(Float, gl_lens_kcube, 0.1f, 0)
-CVAR(Float, gl_lens_chromatic, 1.2f, 0)
+CVAR(Float, gl_lens_chromatic, 1.12f, 0)
 
 EXTERN_CVAR(Float, vid_brightness)
 EXTERN_CVAR(Float, vid_contrast)
@@ -310,12 +310,24 @@ void FGLRenderer::LensDistortScene()
 		0.0f
 	};
 
+	float aspect = mOutputViewport.width / mOutputViewport.height;
+
+	// Scale factor to keep sampling within the input texture
+	float r2 = aspect * aspect * 0.25 + 0.25f;
+	float sqrt_r2 = sqrt(r2);
+	float f0 = 1.0f + MAX(r2 * (k[0] + kcube[0] * sqrt_r2), 0.0f);
+	float f2 = 1.0f + MAX(r2 * (k[2] + kcube[2] * sqrt_r2), 0.0f);
+	float f = MAX(f0, f2);
+	float scale = 1.0f / f;
+
 	mBuffers->BindHudFB();
 	mBuffers->BindSceneTexture(0);
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
 	mLensShader->Bind();
 	mLensShader->InputTexture.Set(0);
+	mLensShader->AspectRatio.Set(aspect);
+	mLensShader->Scale.Set(scale);
 	mLensShader->LensDistortionCoefficient.Set(k);
 	mLensShader->CubicDistortionValue.Set(kcube);
 	mVBO->BindVBO();

src/gl/shaders/gl_lensshader.cpp

@@ -59,6 +59,8 @@ void FLensShader::Bind()
 		mShader.Link("shaders/glsl/lensdistortion");
 		mShader.SetAttribLocation(0, "PositionInProjection");
 		InputTexture.Init(mShader, "InputTexture");
+		AspectRatio.Init(mShader, "Aspect");
+		Scale.Init(mShader, "Scale");
 		LensDistortionCoefficient.Init(mShader, "k");
 		CubicDistortionValue.Init(mShader, "kcube");
 	}

src/gl/shaders/gl_lensshader.h

@@ -9,6 +9,8 @@ public:
 	void Bind();
 
 	FBufferedUniform1i InputTexture;
+	FBufferedUniform1f AspectRatio;
+	FBufferedUniform1f Scale;
 	FBufferedUniform4f LensDistortionCoefficient;
 	FBufferedUniform4f CubicDistortionValue;
 

wadsrc/static/shaders/glsl/lensdistortion.fp

@@ -33,18 +33,21 @@ in vec2 TexCoord;
 out vec4 FragColor;
 
 uniform sampler2D InputTexture;
-uniform vec4 k;     // lens distortion coefficient 
+uniform float Aspect; // image width/height
-uniform vec4 kcube; // cubic distortion value
+uniform float Scale;  // 1/max(f)
+uniform vec4 k;       // lens distortion coefficient 
+uniform vec4 kcube;   // cubic distortion value
 
 void main()
 {
-	vec2 position = TexCoord - vec2(0.5);
+	vec2 position = (TexCoord - vec2(0.5));
 
-	float r2 = dot(position, position);
+	vec2 p = vec2(position.x * Aspect, position.y);
+	float r2 = dot(p, p);
 	vec3 f = vec3(1.0) + r2 * (k.rgb + kcube.rgb * sqrt(r2));
 
-	vec3 x = f * position.x + 0.5;
+	vec3 x = f * position.x * Scale + 0.5;
-	vec3 y = f * position.y + 0.5;
+	vec3 y = f * position.y * Scale + 0.5;
 
 	vec3 c;
 	c.r = texture(InputTexture, vec2(x.r, y.r)).r;