From 45b3ef79c8b1cd5288871309714d3b91069c66ee Mon Sep 17 00:00:00 2001
From: Bill Currie <bill@taniwha.org>
Date: Fri, 27 May 2022 12:21:50 +0900
Subject: [PATCH] [model] Trim decompressed vis sets

I had been trimming for the solid leaf, but not the empty leafs. I had
assumed the vis tool would trim the bits, but it seems to not be
reliable (though it could be a bug in qfvis, I think the map in question
is one of my test maps).
---
 libs/models/brush/model_brush.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/libs/models/brush/model_brush.c b/libs/models/brush/model_brush.c
index 377e871f4..7037710d3 100644
--- a/libs/models/brush/model_brush.c
+++ b/libs/models/brush/model_brush.c
@@ -153,13 +153,13 @@ Mod_LeafPVS (const mleaf_t *leaf, const model_t *model)
 	static set_t *novis;
 	static set_t *decompressed;
 	unsigned    numvis = model->brush.visleafs;
+	unsigned    excess = SET_SIZE (numvis) - numvis;
 
 	if (leaf == model->brush.leafs) {
 		if (!novis) {
 			novis = set_new_size (numvis);
 		}
 		if (!novis->map[0] || SET_SIZE (numvis) > novis->size) {
-			unsigned    excess = SET_SIZE (numvis) - numvis;
 			set_expand (novis, numvis);
 			memset (novis->map, 0xff,
 					SET_WORDS (novis) * sizeof (*novis->map));
@@ -173,6 +173,7 @@ Mod_LeafPVS (const mleaf_t *leaf, const model_t *model)
 	set_expand (decompressed, numvis);
 	Mod_DecompressVis_set (leaf->compressed_vis, &model->brush, 0xff,
 						   decompressed);
+	decompressed->map[SET_WORDS (decompressed) - 1] &= (~SET_ZERO) >> excess;
 	return decompressed;
 }
 
@@ -181,14 +182,16 @@ Mod_LeafPVS_set (const mleaf_t *leaf, const model_t *model, byte defvis,
 				 set_t *out)
 {
 	unsigned    numvis = model->brush.visleafs;
+	unsigned    excess = SET_SIZE (numvis) - numvis;
+
 	set_expand (out, numvis);
 	if (leaf == model->brush.leafs) {
-		unsigned    excess = SET_SIZE (numvis) - numvis;
 		memset (out->map, defvis, SET_WORDS (out) * sizeof (*out->map));
 		out->map[SET_WORDS (out) - 1] &= (~SET_ZERO) >> excess;
 		return;
 	}
 	Mod_DecompressVis_set (leaf->compressed_vis, &model->brush, defvis, out);
+	out->map[SET_WORDS (out) - 1] &= (~SET_ZERO) >> excess;
 }
 
 VISIBLE void
@@ -196,9 +199,10 @@ Mod_LeafPVS_mix (const mleaf_t *leaf, const model_t *model, byte defvis,
 				 set_t *out)
 {
 	unsigned    numvis = model->brush.visleafs;
+	unsigned    excess = SET_SIZE (numvis) - numvis;
+
 	set_expand (out, numvis);
 	if (leaf == model->brush.leafs) {
-		unsigned    excess = SET_SIZE (numvis) - numvis;
 		byte       *o = (byte *) out->map;
 		for (int i = SET_WORDS (out) * sizeof (*out->map); i-- > 0; ) {
 			*o++ |= defvis;
@@ -207,6 +211,7 @@ Mod_LeafPVS_mix (const mleaf_t *leaf, const model_t *model, byte defvis,
 		return;
 	}
 	Mod_DecompressVis_mix (leaf->compressed_vis, &model->brush, defvis, out);
+	out->map[SET_WORDS (out) - 1] &= (~SET_ZERO) >> excess;
 }
 
 // BRUSHMODEL LOADING =========================================================