[qfvis] Write out the fat-pvs file

The output fat-pvs data is the *difference* between the base pvs and fat
pvs. This currently makes for about 64kB savings for marcher.bsp, and
about 233MB savings for ad_tears.bsp (or about 50% (470.7MB->237.1MB)).
I expect using utf-8 encoding for the run lengths to make for even
bigger savings (the second output fat-pvs leaf of marcher.bsp is all 0s,
or 6 bytes in the file, which would reduce to 3 bytes using utf-8).

include/QF/pvsfile.h

@@ -0,0 +1,64 @@
+/*
+	pvsfile.h
+
+	External pvs file support
+
+	Copyright (C) 2021 Bill Currie <bill@taniwha.org>
+
+	Author: Bill Currie <bill@taniwha.org>
+	Date: 2021/07/27
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation; either version 2
+	of the License, or (at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+	See the GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to:
+
+		Free Software Foundation, Inc.
+		59 Temple Place - Suite 330
+		Boston, MA  02111-1307, USA
+
+*/
+
+#ifndef __QF_pvsfile_h
+#define __QF_pvsfile_h
+
+#include <stdint.h>
+
+#include "QF/qtypes.h"
+
+/** \defgroup pvsfile External PVS
+	\ingroup utils
+
+	External PVS data.
+*/
+///@{
+
+#define PVS_MAGIC "QF PVS\r\n\x1a\x04"
+#define PVS_VERSION 1
+
+#define PVS_IS_FATPVS 1	///< data is for fat pvs (phs, lighting, etc)
+						///< otherwise it's the base pvs
+#define PVS_UTF8_RLE  2	///< 0-byte run length encoding uses utf-8
+
+typedef struct pvsfile_s {
+	char        magic[16];
+
+	uint32_t    version;
+	uint32_t    md4_offset;		///< offset of md4 sum, or 0 if not present
+	uint32_t    flags;
+	uint32_t    visleafs;		///< does NOT include leaf-0
+	uint32_t    visoffsets[];	///< does NOT include leaf-0
+} pvsfile_t;
+
+///@}
+
+#endif//__QF_pvsfile_h

tools/qfvis/include/vis.h

@@ -41,6 +41,7 @@
 #include <pthread.h>
 extern pthread_rwlock_t *global_lock;
 extern pthread_rwlock_t *portal_locks;
+extern pthread_rwlock_t *stats_lock;
 
 #define	WRLOCK(l) \
 	do { \
@@ -213,6 +214,8 @@ void PortalBase (basethread_t *thread, portal_t *portal);
 void PortalFlow (threaddata_t *data, portal_t *portal);
 void CalcAmbientSounds (void);
 
+int CompressRow (byte *dest, const byte *vis, unsigned num_leafs);
+
 void CalcFatPVS (void);
 
 void RunThreads (void *(*thread_func) (void *), int (*progress)(int, int));

tools/qfvis/source/fatpvs.c

@@ -28,10 +28,14 @@
 # include "config.h"
 #endif
 
+#include <string.h>
 #include <stdlib.h>
 
 #include "QF/bspfile.h"
+#include "QF/pvsfile.h"
+#include "QF/quakefs.h"
 #include "QF/set.h"
+#include "QF/sizebuf.h"
 #include "QF/sys.h"
 
 #include "tools/qfvis/include/options.h"
@@ -40,10 +44,30 @@
 static set_pool_t *set_pool;
 static set_t *base_pvs;
 static set_t *fat_pvs;
+static sizebuf_t *cmp_pvs;
 
 static unsigned num_leafs;
+static unsigned visbytes;
 static unsigned work_leaf;
 
+typedef struct {
+	long        pvs_visible;
+	long        fat_visible;
+	long        fat_bytes;
+} fatstats_t;
+
+fatstats_t  fatstats;
+
+static void
+update_stats (fatstats_t *stats)
+{
+	WRLOCK (stats_lock);
+	fatstats.pvs_visible += stats->pvs_visible;
+	fatstats.fat_visible += stats->fat_visible;
+	fatstats.fat_bytes += stats->fat_bytes;
+	UNLOCK (stats_lock);
+}
+
 static int
 print_progress (int prev_prog, int spinner_ind)
 {
@@ -103,8 +127,8 @@ decompress_vis (const byte *in, unsigned numleafs, set_t *pvs)
 static void *
 decompress_thread (void *d)
 {
+	fatstats_t  stats = { };
 	int         thread = (intptr_t) d;
-	int64_t     count = 0;
 
 	while (1) {
 		unsigned    leaf_num = next_leaf ();
@@ -112,7 +136,7 @@ decompress_thread (void *d)
 		if (working)
 			working[thread] = leaf_num;
 		if (leaf_num == ~0u) {
-			return 0;
+			break;
 		}
 		byte       *visdata = 0;
 		dleaf_t    *leaf = &bsp->leafs[leaf_num];
@@ -123,20 +147,17 @@ decompress_thread (void *d)
 		if (leaf_num == 0) {
 			continue;
 		}
-		for (set_iter_t *iter = set_first_r (&set_pool[thread],
-											 &base_pvs[leaf_num]);
-			 iter;
-			 iter = set_next_r (&set_pool[thread], iter)) {
-			count++;
-		}
+		stats.pvs_visible += set_count (&base_pvs[leaf_num]);
 	}
+	update_stats (&stats);
+	return 0;
 }
 
 static void *
 fatten_thread (void *d)
 {
+	fatstats_t  stats = { };
 	int         thread = (intptr_t) d;
-	int64_t     count = 0;
 
 	while (1) {
 		unsigned    leaf_num = next_leaf ();
@@ -144,7 +165,7 @@ fatten_thread (void *d)
 		if (working)
 			working[thread] = leaf_num;
 		if (leaf_num == ~0u) {
-			return 0;
+			break;
 		}
 
 		set_assign (&fat_pvs[leaf_num], &base_pvs[leaf_num]);
@@ -160,13 +181,11 @@ fatten_thread (void *d)
 		if (leaf_num == 0) {
 			continue;
 		}
-		for (set_iter_t *iter = set_first_r (&set_pool[thread],
-											 &base_pvs[leaf_num]);
-			 iter;
-			 iter = set_next_r (&set_pool[thread], iter)) {
-			count++;
-		}
+		stats.fat_visible += set_count (&fat_pvs[leaf_num]);
+		set_difference (&fat_pvs[leaf_num], &base_pvs[leaf_num]);
 	}
+	update_stats (&stats);
+	return 0;
 }
 #if 0
 static void *
@@ -196,19 +215,82 @@ fatten_thread2 (void *d)
 	}
 }
 #endif
+
+static void *
+compress_thread (void *d)
+{
+	fatstats_t  stats = { };
+	int         thread = (intptr_t) d;
+	byte        compressed[(visbytes * 3) / 2];
+
+	while (1) {
+		unsigned    leaf_num = next_leaf ();
+
+		if (working)
+			working[thread] = leaf_num;
+		if (leaf_num == ~0u) {
+			break;
+		}
+		const byte *fat_bytes = (const byte *) fat_pvs[leaf_num].map;
+		int         cmp_bytes = CompressRow (compressed, fat_bytes, num_leafs);
+		SZ_Write (&cmp_pvs[leaf_num], compressed, cmp_bytes);
+		stats.fat_bytes += cmp_bytes;
+	}
+	update_stats (&stats);
+	return 0;
+}
+
 void
 CalcFatPVS (void)
 {
 	set_pool = calloc (options.threads, sizeof (set_pool_t));
 	num_leafs = bsp->models[0].visleafs;
+	visbytes = (num_leafs + 7) / 8;
 	base_pvs = malloc (num_leafs * sizeof (set_t));
 	fat_pvs = malloc (num_leafs * sizeof (set_t));
+	cmp_pvs = malloc (num_leafs * sizeof (sizebuf_t));
 	for (unsigned i = 0; i < num_leafs; i++) {
 		base_pvs[i] = (set_t) SET_STATIC_INIT (num_leafs, malloc);
 		fat_pvs[i] = (set_t) SET_STATIC_INIT (num_leafs, malloc);
+		cmp_pvs[i] = (sizebuf_t) {
+			.data = malloc ((visbytes * 3) / 2),
+			.maxsize = (visbytes * 3) / 2,
+		};
 	}
 	work_leaf = 0;
 	RunThreads (decompress_thread, print_progress);
 	work_leaf = 0;
 	RunThreads (fatten_thread, print_progress);
+	work_leaf = 0;
+	RunThreads (compress_thread, print_progress);
+	printf ("Average leafs visible / fat visible / total: %d / %d / %d\n",
+			(int) (fatstats.pvs_visible / num_leafs),
+			(int) (fatstats.fat_visible / num_leafs), num_leafs);
+	printf ("Compressed fat vis size: %ld\n", fatstats.fat_bytes);
+
+	uint32_t    offset = sizeof (pvsfile_t) + num_leafs * sizeof (uint32_t);
+	pvsfile_t  *pvsfile = malloc (offset + fatstats.fat_bytes);
+
+	strncpy (pvsfile->magic, PVS_MAGIC, sizeof (pvsfile->magic));
+	pvsfile->version = PVS_VERSION;
+	pvsfile->md4_offset = 0;	//FIXME add
+	pvsfile->flags = PVS_IS_FATPVS;
+	pvsfile->visleafs = num_leafs;
+	for (unsigned i = 0; i < num_leafs; i++) {
+		unsigned    size = cmp_pvs[i].cursize;
+		pvsfile->visoffsets[i] = offset;
+		memcpy ((byte *) pvsfile + offset, cmp_pvs[i].data, size);
+		offset += size;
+	}
+
+	dstring_t  *pvsname = dstring_new ();
+	dstring_copystr (pvsname, options.bspfile->str);
+	QFS_SetExtension (pvsname, ".pvs");
+
+	QFile      *f = Qopen (pvsname->str, "wb");
+	if (!f) {
+		Sys_Error ("couldn't open %s for writing.", pvsname->str);
+	}
+	Qwrite (f, pvsfile, offset);
+	Qclose (f);
 }

tools/qfvis/source/qfvis.c

@@ -442,7 +442,7 @@ UpdateMightsee (threaddata_t *thread, cluster_t *source, cluster_t *dest)
 }
 
 static void
-UpdateStates (threaddata_t *thread)
+UpdateStats (threaddata_t *thread)
 {
 	WRLOCK (stats_lock);
 	global_stats.portaltest += thread->stats.portaltest;
@@ -512,7 +512,7 @@ PortalCompleted (threaddata_t *thread, portal_t *completed)
 	}
 	set_delete_r (&thread->set_pool, changed);
 
-	UpdateStates (thread);
+	UpdateStats (thread);
 }
 
 static void
@@ -734,10 +734,12 @@ WatchThread (void *_wd)
 			}
 		}
 	}
-	if (options.verbosity >= 4)
+	if (options.verbosity >= 4) {
 		printf ("watch thread done\n");
-	else if (options.verbosity >= 0)
+	} else if (options.verbosity >= 0) {
+		prev_prog = wd->progress (prev_prog, spinner_ind);
 		printf ("\n");
+	}
 	free (local_work);
 
 	return NULL;
@@ -781,14 +783,14 @@ RunThreads (void *(*thread_func) (void *), int (*progress)(int, int))
 #endif
 }
 
-static int
-CompressRow (byte *vis, byte *dest)
+int
+CompressRow (byte *dest, const byte *vis, unsigned num_leafs)
 {
 	int         rep, visrow, j;
 	byte       *dest_p;
 
 	dest_p = dest;
-	visrow = (numrealleafs + 7) >> 3;
+	visrow = (num_leafs + 7) >> 3;
 
 	for (j = 0; j < visrow; j++) {
 		*dest_p++ = vis[j];
@@ -870,7 +872,7 @@ ClusterFlow (int clusternum)
 		printf ("cluster %4i : %4i visible\n", clusternum, numvis);
 	totalvis += numvis;
 
-	i = CompressRow (outbuffer, compressed);
+	i = CompressRow (compressed, outbuffer, numrealleafs);
 	cluster->visofs = visdata->size;
 	dstring_append (visdata, (char *) compressed, i);
 }