From 946867c82e3e912d3f8ca7d9b4d9aa61c01e52ff Mon Sep 17 00:00:00 2001 From: Bill Currie Date: Mon, 26 Jul 2021 22:42:03 +0900 Subject: [PATCH] [qfvis] Start work on an off-line fat pvs compiler Extremely large maps take a very long time to process their PVS sets for PHS or shadows, so having an off-line compiler seems like a good idea. The data isn't written out yet, and the fat pvs code may not be optimal for cache access, but it gets through ad_tears in about 500s (12 threads, compared to 2100s single-threaded in the qw server). --- tools/qfvis/include/options.h | 2 + tools/qfvis/include/vis.h | 8 ++ tools/qfvis/source/Makemodule.am | 1 + tools/qfvis/source/fatpvs.c | 214 +++++++++++++++++++++++++++++++ tools/qfvis/source/options.c | 11 ++ tools/qfvis/source/qfvis.c | 114 +++++++++------- 6 files changed, 304 insertions(+), 46 deletions(-) create mode 100644 tools/qfvis/source/fatpvs.c diff --git a/tools/qfvis/include/options.h b/tools/qfvis/include/options.h index cd6349aa8..4af2c67ac 100644 --- a/tools/qfvis/include/options.h +++ b/tools/qfvis/include/options.h @@ -34,6 +34,8 @@ typedef struct { int verbosity; // 0=silent int threads; qboolean minimal; + qboolean no_auto_pvs; + qboolean fat_pvs; int level; size_t portal_limit; struct dstring_s *bspfile; diff --git a/tools/qfvis/include/vis.h b/tools/qfvis/include/vis.h index 9367760b4..c2c1edfed 100644 --- a/tools/qfvis/include/vis.h +++ b/tools/qfvis/include/vis.h @@ -213,4 +213,12 @@ void PortalBase (basethread_t *thread, portal_t *portal); void PortalFlow (threaddata_t *data, portal_t *portal); void CalcAmbientSounds (void); +void CalcFatPVS (void); + +void RunThreads (void *(*thread_func) (void *), int (*progress)(int, int)); + +extern const char spinner[]; +extern const char progress[]; +extern int *working; + #endif// __vis_h diff --git a/tools/qfvis/source/Makemodule.am b/tools/qfvis/source/Makemodule.am index cb11f2406..7efb7bbe2 100644 --- a/tools/qfvis/source/Makemodule.am +++ b/tools/qfvis/source/Makemodule.am @@ -7,6 +7,7 @@ bin_PROGRAMS += @QFVIS_TARGETS@ qfvis_SOURCES = \ tools/qfvis/source/base-vis.c \ + tools/qfvis/source/fatpvs.c \ tools/qfvis/source/flow.c \ tools/qfvis/source/options.c \ tools/qfvis/source/qfvis.c \ diff --git a/tools/qfvis/source/fatpvs.c b/tools/qfvis/source/fatpvs.c new file mode 100644 index 000000000..749f540d3 --- /dev/null +++ b/tools/qfvis/source/fatpvs.c @@ -0,0 +1,214 @@ +/* + fatpvs.c + + PVS PHS generator tool + + Copyright (C) 2021 Bil Currie + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to: + + Free Software Foundation, Inc. + 59 Temple Place - Suite 330 + Boston, MA 02111-1307, USA + +*/ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include + +#include "QF/bspfile.h" +#include "QF/set.h" +#include "QF/sys.h" + +#include "tools/qfvis/include/options.h" +#include "tools/qfvis/include/vis.h" + +static set_pool_t *set_pool; +static set_t *base_pvs; +static set_t *fat_pvs; + +static unsigned num_leafs; +static unsigned work_leaf; + +static int +print_progress (int prev_prog, int spinner_ind) +{ + int prog; + prog = work_leaf * 50 / num_leafs; + if (prog > prev_prog) + printf ("%.*s", prog - prev_prog, progress + prev_prog); + printf (" %c\b\b", spinner[spinner_ind % 4]); + fflush (stdout); + return prog; +} + +static unsigned +next_leaf (void) +{ + unsigned leaf = ~0; + WRLOCK (global_lock); + if (work_leaf < num_leafs) { + leaf = work_leaf++; + } + UNLOCK (global_lock); + return leaf; +} + +static inline void +decompress_vis (const byte *in, unsigned numleafs, set_t *pvs) +{ + byte *out = (byte *) pvs->map; + byte *start = out; + int row, c; + + row = (numleafs + 7) >> 3; + + if (!in) { // no vis info, so make all visible + while (row) { + *out++ = 0xff; + row--; + } + return; + } + + do { + if (*in) { + *out++ = *in++; + continue; + } + + c = in[1]; + in += 2; + while (c) { + *out++ = 0; + c--; + } + } while (out - start < row); +} + +static void * +decompress_thread (void *d) +{ + int thread = (intptr_t) d; + int64_t count = 0; + + while (1) { + unsigned leaf_num = next_leaf (); + + if (working) + working[thread] = leaf_num; + if (leaf_num == ~0u) { + return 0; + } + byte *visdata = 0; + dleaf_t *leaf = &bsp->leafs[leaf_num]; + if (leaf->visofs >= 0) { + visdata = bsp->visdata + leaf->visofs; + } + decompress_vis (visdata, num_leafs, &base_pvs[leaf_num]); + if (leaf_num == 0) { + continue; + } + for (set_iter_t *iter = set_first_r (&set_pool[thread], + &base_pvs[leaf_num]); + iter; + iter = set_next_r (&set_pool[thread], iter)) { + count++; + } + } +} + +static void * +fatten_thread (void *d) +{ + int thread = (intptr_t) d; + int64_t count = 0; + + while (1) { + unsigned leaf_num = next_leaf (); + + if (working) + working[thread] = leaf_num; + if (leaf_num == ~0u) { + return 0; + } + + set_assign (&fat_pvs[leaf_num], &base_pvs[leaf_num]); + for (set_iter_t *iter = set_first_r (&set_pool[thread], + &base_pvs[leaf_num]); + iter; + iter = set_next_r (&set_pool[thread], iter)) { + + // or this pvs row into the fat pvs + // +1 because pvs is 1 based + set_union (&fat_pvs[leaf_num], &base_pvs[iter->element + 1]); + } + if (leaf_num == 0) { + continue; + } + for (set_iter_t *iter = set_first_r (&set_pool[thread], + &base_pvs[leaf_num]); + iter; + iter = set_next_r (&set_pool[thread], iter)) { + count++; + } + } +} +#if 0 +static void * +fatten_thread2 (void *d) +{ + for (unsigned visword = 0; visword < SET_WORDS (&sv.pvs[0]); visword++) { + Sys_Printf ("%d / %ld\n", visword, SET_WORDS (&sv.pvs[0])); + for (i = 0; i < num; i++) { + for (unsigned j = 0; j < SET_BITS; j++) { + unsigned visbit = visword * SET_BITS + j; + if (visbit >= (unsigned) num) { + break; + } + if (SET_TEST_MEMBER (&sv.pvs[i], visbit)) { + // or this pvs row into the phs + // +1 because pvs is 1 based + set_union (&sv.phs[i], &sv.pvs[visbit + 1]); + } + } + } + } + for (i = 1; i < num; i++) { + for (set_iter_t *iter = set_first (&sv.phs[i]); iter; + iter = set_next (iter)) { + count++; + } + } +} +#endif +void +CalcFatPVS (void) +{ + set_pool = calloc (options.threads, sizeof (set_pool_t)); + num_leafs = bsp->models[0].visleafs; + base_pvs = malloc (num_leafs * sizeof (set_t)); + fat_pvs = malloc (num_leafs * sizeof (set_t)); + for (unsigned i = 0; i < num_leafs; i++) { + base_pvs[i] = (set_t) SET_STATIC_INIT (num_leafs, malloc); + fat_pvs[i] = (set_t) SET_STATIC_INIT (num_leafs, malloc); + } + work_leaf = 0; + RunThreads (decompress_thread, print_progress); + work_leaf = 0; + RunThreads (fatten_thread, print_progress); +} diff --git a/tools/qfvis/source/options.c b/tools/qfvis/source/options.c index e66eca278..291f67c63 100644 --- a/tools/qfvis/source/options.c +++ b/tools/qfvis/source/options.c @@ -49,9 +49,13 @@ const char *this_program; enum { start_opts = 255, // not used, starts the enum. OPT_PORTAL_LIMIT, + OPT_FAT_PVS, + OPT_FULL_PVS, }; static struct option const long_options[] = { + {"fat-pvs", no_argument, 0, OPT_FAT_PVS}, + {"full-pvs", no_argument, 0, OPT_FULL_PVS}, {"quiet", no_argument, 0, 'q'}, {"verbose", no_argument, 0, 'v'}, {"help", no_argument, 0, 'h'}, @@ -140,6 +144,13 @@ DecodeArgs (int argc, char **argv) case OPT_PORTAL_LIMIT: options.portal_limit = atoi (optarg); break; + case OPT_FAT_PVS: + options.fat_pvs = true; + options.no_auto_pvs = true; + break; + case OPT_FULL_PVS: + options.fat_pvs = true; + break; case 'm': // minimal vis options.minimal = true; break; diff --git a/tools/qfvis/source/qfvis.c b/tools/qfvis/source/qfvis.c index b6e0f3248..e66d8e436 100644 --- a/tools/qfvis/source/qfvis.c +++ b/tools/qfvis/source/qfvis.c @@ -693,10 +693,16 @@ print_progress (int prev_prog, int spinner_ind) return prog; } +typedef struct { + int thread; + int (*progress)(int, int); +} watch_data_t; + static void * -WatchThread (void *_thread) +WatchThread (void *_wd) { - int thread = (intptr_t) _thread; + watch_data_t *wd = _wd; + int thread = wd->thread; int *local_work = malloc (thread * sizeof (int)); int i; int spinner_ind = 0; @@ -720,7 +726,7 @@ WatchThread (void *_thread) if (options.verbosity >= 4) print_thread_stats (local_work, thread, spinner_ind); else if (options.verbosity >= 0) - prev_prog = print_progress (prev_prog, spinner_ind); + prev_prog = wd->progress (prev_prog, spinner_ind); if (prev_port != portal_count || stalled++ == 10) { prev_port = portal_count; stalled = 0; @@ -738,8 +744,8 @@ WatchThread (void *_thread) } #endif -static void -RunThreads (void *(*thread_func) (void *)) +void +RunThreads (void *(*thread_func) (void *), int (*progress)(int, int)) { #ifdef USE_PTHREADS pthread_t *work_threads; @@ -754,8 +760,9 @@ RunThreads (void *(*thread_func) (void *)) thread_func, (void *) (intptr_t) i) == -1) Sys_Error ("pthread_create failed"); } + watch_data_t wd = { i, progress }; if (pthread_create (&work_threads[i], &threads_attrib, - WatchThread, (void *) (intptr_t) i) == -1) + WatchThread, &wd) == -1) Sys_Error ("pthread_create failed"); for (i = 0; i < options.threads; i++) { @@ -770,7 +777,7 @@ RunThreads (void *(*thread_func) (void *)) thread_func (0); } #else - LeafThread (0); + thread_func (0); #endif } @@ -887,7 +894,7 @@ BasePortalVis (void) printf ("\n"); start = Sys_DoubleTime (); - RunThreads (BaseVisThread); + RunThreads (BaseVisThread, print_progress); end = Sys_DoubleTime (); if (options.verbosity >= 1) { @@ -924,7 +931,7 @@ CalcPortalVis (void) portal_count = 0; - RunThreads (LeafThread); + RunThreads (LeafThread, print_progress); if (options.verbosity >= 1) { printf ("portalcheck: %i portaltest: %i portalpass: %i\n", @@ -1343,35 +1350,12 @@ LoadPortals (char *name) Qclose (f); } -int -main (int argc, char **argv) +static void +generate_pvs (void) { - double start, stop; - dstring_t *portalfile = dstring_new (); QFile *f; - main_thread.memsuper = new_memsuper (); - - start = Sys_DoubleTime (); - - this_program = argv[0]; - - DecodeArgs (argc, argv); - - InitThreads (); - - if (!options.bspfile) { - usage (1); - Sys_Error ("%s: no bsp file specified.", this_program); - } - - QFS_SetExtension (options.bspfile, ".bsp"); - - f = Qopen (options.bspfile->str, "rb"); - if (!f) - Sys_Error ("couldn't open %s for reading.", options.bspfile->str); - bsp = LoadBSPFile (f, Qfilesize (f)); - Qclose (f); + dstring_t *portalfile = dstring_new (); visdata = dstring_new (); @@ -1400,21 +1384,59 @@ main (int argc, char **argv) WriteBSPFile (bsp, f); Qclose (f); + dstring_delete (portalfile); + dstring_delete (visdata); + dstring_delete (options.bspfile); + free (leafcluster); + free (uncompressed); + free (portals); + free (clusters); +} + +int +main (int argc, char **argv) +{ + double start, stop; + QFile *f; + + main_thread.memsuper = new_memsuper (); + + start = Sys_DoubleTime (); + + this_program = argv[0]; + + DecodeArgs (argc, argv); + + InitThreads (); + + if (!options.bspfile) { + usage (1); + Sys_Error ("%s: no bsp file specified.", this_program); + } + + QFS_SetExtension (options.bspfile, ".bsp"); + + f = Qopen (options.bspfile->str, "rb"); + if (!f) + Sys_Error ("couldn't open %s for reading.", options.bspfile->str); + bsp = LoadBSPFile (f, Qfilesize (f)); + Qclose (f); + + if (!options.no_auto_pvs) { + generate_pvs (); + } + if (options.fat_pvs) { + CalcFatPVS (); + } + + BSP_Free (bsp); + + EndThreads (); + stop = Sys_DoubleTime (); if (options.verbosity >= -1) printf ("%5.1f seconds elapsed\n", stop - start); - dstring_delete (portalfile); - dstring_delete (visdata); - dstring_delete (options.bspfile); - BSP_Free (bsp); - free (leafcluster); - free (uncompressed); - free (portals); - free (clusters); - - EndThreads (); - return 0; }