"Asynchronous" file queue calls findmultiplefiles on another thread, then reads the file on the main thread

This replaces the exec command's threading from before.
(See d0e7ba309a59580988225816d104e343b64d67d0.) The order of exec commands
alone is guranteed. And it removes the need to put mutexes on the command
buffer. I plan to use this system for addfile too...

src/Makefile

@@ -484,6 +484,7 @@ DBGNAME?=$(EXENAME).debug
 OBJS:=$(i_main_o) \
 		$(OBJDIR)/comptime.o \
 		$(OBJDIR)/string.o   \
+		$(OBJDIR)/d_async.o  \
 		$(OBJDIR)/d_main.o   \
 		$(OBJDIR)/d_clisrv.o \
 		$(OBJDIR)/d_net.o    \

src/command.c

@@ -18,6 +18,7 @@
 
 #include "doomdef.h"
 #include "doomstat.h"
+#include "d_main.h"
 #include "command.h"
 #include "console.h"
 #include "z_zone.h"
@@ -34,6 +35,8 @@
 #include "p_setup.h"
 #include "lua_script.h"
 #include "d_netfil.h" // findfile
+#include "d_async.h"
+#include "filesrch.h"
 #include "i_threads.h"
 
 #ifdef HAVE_THREADS
@@ -695,7 +698,6 @@ static void COM_CEchoDuration_f(void)
 
 struct COM_Exec_Ctx
 {
-	char    * filename;
 	boolean   noerror;
 	boolean   silent;
 	UINT8   * buf;
@@ -704,47 +706,25 @@ struct COM_Exec_Ctx
 static void Free_COM_Exec_Ctx (struct COM_Exec_Ctx *ctx)
 {
 	Z_Free(ctx->buf);
-	free(ctx->filename);
-	free(ctx);
+	Z_Free(ctx);
 }
 
-static void COM_Exec_Thread (struct COM_Exec_Ctx *ctx)
+void COM_ExecuteFile (void *p, void *u)
 {
-	char filename[256];
+	filequery_t         * q   = p;
+	struct COM_Exec_Ctx * ctx = u;
 
 	ctx->buf = NULL;
 
-	// load file
-	// Try with Argv passed verbatim first, for back compat
-	FIL_ReadFile(ctx->filename, &ctx->buf);
-
-#ifdef HAVE_THREADS
-	if (I_thread_is_stopped())
+	if (q->status == FS_FOUND)
 	{
-		return Free_COM_Exec_Ctx(ctx);
-	}
-#endif
-
-	if (! ctx->buf)
-	{
-		// Now try by searching the file path
-		// filename is modified with the full found path
-		strcpy(filename, ctx->filename);
-		if (findfile(filename, NULL, true) != FS_NOTFOUND)
-			FIL_ReadFile(filename, &ctx->buf);
-
-#ifdef HAVE_THREADS
-		if (I_thread_is_stopped())
-		{
-			return Free_COM_Exec_Ctx(ctx);
-		}
-#endif
+		FIL_ReadFile(q->filename, &ctx->buf);
 	}
 
 	if (ctx->buf)
 	{
 		if (! ctx->silent)
-			CONS_Printf(M_GetText("executing %s\n"), ctx->filename);
+			CONS_Printf(M_GetText("executing %s\n"), q->filename);
 
 		// insert text file into the command buffer
 		COM_BufAddText((char *)ctx->buf);
@@ -753,7 +733,7 @@ static void COM_Exec_Thread (struct COM_Exec_Ctx *ctx)
 	else
 	{
 		if (! ctx->noerror)
-			CONS_Printf(M_GetText("couldn't execute file %s\n"), ctx->filename);
+			CONS_Printf(M_GetText("couldn't execute file %s\n"), q->filename);
 	}
 
 	Free_COM_Exec_Ctx(ctx);
@@ -763,32 +743,48 @@ static void COM_Exec_Thread (struct COM_Exec_Ctx *ctx)
   */
 static void COM_Exec_f(void)
 {
+	char filenamebuf[MAX_WADPATH];
+
 	struct COM_Exec_Ctx *ctx;
 
+	const char * filename;
+
+	filequery_t q;
+
 	if (COM_Argc() < 2 || COM_Argc() > 3)
 	{
 		CONS_Printf(M_GetText("exec <filename>: run a script file\n"));
 		return;
 	}
 
-	ctx = malloc(sizeof *ctx);
+	filename = COM_Argv(1);
 
-	ctx->filename = strdup(COM_Argv(1));
+	ctx = ZZ_Alloc(sizeof *ctx);
 
 	ctx->noerror  = COM_CheckParm("-noerror");
 	ctx->silent   = COM_CheckParm("-silent");
 
-	if (COM_CheckParm("-singlethread"))
+	if (COM_CheckParm("-sync"))
 	{
-		COM_Exec_Thread(ctx);
+		q.filename = filenamebuf;
+
+		strcpy(q.filename, filename);
+
+		if (FIL_FileOK(va(pandf,srb2home,filename)))
+		{
+			q.status = FS_FOUND;
+		}
+		else
+		{
+			q.status = findfile(q.filename, NULL, true);
+		}
+
+		COM_ExecuteFile(&q, ctx);
 	}
 	else
 	{
-#ifdef HAVE_THREADS
-		I_spawn_thread("exec-command", (I_thread_fn)COM_Exec_Thread, ctx);
-#else
-		COM_Exec_Thread(ctx);
-#endif
+		q.filename = strcpy(ZZ_Alloc(MAX_WADPATH), filename);
+		Append_async_addfile(ASYNC_EXEC, q.filename, ctx);
 	}
 
 }

src/command.h

@@ -45,6 +45,9 @@ void COM_ImmedExecute(const char *ptext);
 // Execute commands in buffer, flush them
 void COM_BufExecute(void);
 
+// Execute a whole file (internal use only :v)
+void COM_ExecuteFile(void *query, void *ctx);
+
 // As above; and progress the wait timer.
 void COM_BufTicker(void);
 

src/d_async.c

@@ -0,0 +1,259 @@
+// SONIC ROBO BLAST 2 KART
+//-----------------------------------------------------------------------------
+// Copyright (C) 2020 by James R.
+//
+// This program is free software distributed under the
+// terms of the GNU General Public License, version 2.
+// See the 'LICENSE' file for more details.
+//-----------------------------------------------------------------------------
+/// \file  d_async.c
+/// \brief Add wad/exec config asynchronously to the main thread, but in order
+
+#include "doomdef.h"
+#include "d_main.h"
+#include "d_async.h"
+#include "filesrch.h"
+#include "i_threads.h"
+#include "m_misc.h"
+#include "p_setup.h"
+#include "z_zone.h"
+
+struct filename_queue;
+struct async_addfile;
+
+typedef struct filename_queue filename_queue_t;
+typedef struct async_addfile async_addfile_t;
+
+struct filename_queue
+{
+	int                type;
+	char             * name;
+	void             * u;/* exec command has some parameters that need know */
+	filename_queue_t * next;
+};
+
+struct async_addfile
+{
+	int               done;
+	int               query_size;
+	int               query_position;
+	filequery_t     * query;
+	int             * type;
+	void           ** u;
+	async_addfile_t * next;
+};
+
+static filename_queue_t * queue_head;
+static filename_queue_t * queue_tail;
+static int                queue_size;
+
+static I_mutex            async_mutex;
+
+static async_addfile_t  * async_head;
+static async_addfile_t  * async_tail;
+
+static void
+Async_addfile_thread (async_addfile_t *file)
+{
+	findmultiplefiles(file->query_size, file->query, false, true, &async_mutex);
+
+	I_lock_mutex(&async_mutex);
+	{
+		file->done = 1;
+	}
+	I_unlock_mutex(async_mutex);
+}
+
+static void
+Do_addfile (int type, filequery_t *q, void *ctx)
+{
+	switch (type)
+	{
+		case ASYNC_EXEC:
+			COM_ExecuteFile(q, ctx);
+			break;
+	}
+}
+
+void
+Finish_async_addfile (void)
+{
+	async_addfile_t * file = async_head;
+	async_addfile_t * next;
+
+	filequery_t * q;
+
+	filestatus_t status;
+
+	int type;
+	int done;
+
+	while (file != NULL)
+	{
+		while (file->query_position < file->query_size)
+		{
+			q    = &file->query [file->query_position];
+			type =  file->type  [file->query_position];
+
+			I_lock_mutex(&async_mutex);
+			{
+				status = q->status;
+			}
+			I_unlock_mutex(async_mutex);
+
+			if (status == FS_NOTCHECKED)
+			{
+				break;
+			}
+			else
+			{
+				Do_addfile(type, q, file->u[file->query_position]);
+			}
+
+			Z_Free(q->filename);
+
+			file->query_position++;
+		}
+
+		if (file->query_position < file->query_size)
+		{
+			break;
+		}
+
+		I_lock_mutex(&async_mutex);
+		{
+			done = file->done;
+		}
+		I_unlock_mutex(async_mutex);
+
+		if (done)
+		{
+			next = file->next;
+
+			Z_Free(file->u);
+			Z_Free(file->type);
+			Z_Free(file->query);
+			Z_Free(file);
+
+			file = next;
+		}
+		else
+		{
+			break;
+		}
+	}
+
+	async_head = file;
+
+	if (file == NULL)
+	{
+		async_tail = NULL;
+	}
+}
+
+void
+Append_async_addfile (int type, const char * filename, void * userdata)
+{
+	char filenamebuf[MAX_WADPATH];
+
+	filename_queue_t * q;
+
+	filequery_t file;
+
+	if (FIL_FileOK(va(pandf,srb2home,filename)))
+	{
+		file.status   = FS_FOUND;
+		file.filename = filenamebuf;
+
+		strcpy(file.filename, filename);
+
+		Do_addfile(type, &file, userdata);
+	}
+	else
+	{
+		q = ZZ_Alloc(sizeof *q);
+
+		if (queue_tail != NULL)
+		{
+			queue_tail->next = q;
+		}
+		else
+		{
+			queue_head = q;
+		}
+
+		queue_tail = q;
+		queue_size++;
+
+		q->type = type;
+		q->name = strcpy(ZZ_Alloc(MAX_WADPATH), filename);
+		q->u    = userdata;
+		q->next = NULL;
+	}
+}
+
+void
+Detach_async_addfile (void)
+{
+	filename_queue_t * q = queue_head;
+	filename_queue_t * next;
+
+	async_addfile_t  * file;
+
+	filequery_t      * query;
+	int              * type;
+	void            ** udata;
+
+	if (q != NULL)
+	{
+		file = ZZ_Alloc(sizeof *file);
+
+		if (async_tail != NULL)
+		{
+			async_tail->next = file;
+		}
+		else
+		{
+			async_head = file;
+		}
+
+		async_tail = file;
+
+		file->done = 0;
+		file->next = NULL;
+
+		file->query_size     = queue_size;
+		file->query_position = 0;
+
+		file->query = ZZ_Alloc(queue_size * sizeof (*file->query));
+		file->type  = ZZ_Alloc(queue_size * sizeof (*file->type));
+		file->u     = ZZ_Alloc(queue_size * sizeof (*file->u));
+
+		query = file->query;
+		type  = file->type;
+		udata = file->u;
+
+		do
+		{
+			*type++  = q->type;
+			*udata++ = q->u;
+
+			query->status       = FS_NOTCHECKED;
+			query->wantedmd5sum = NULL;
+			query->filename     = q->name;
+			query++;
+
+			next = q->next;
+
+			Z_Free(q);
+		}
+		while (( q = next ) != NULL) ;
+
+		I_spawn_thread("async-addfile",
+				(I_thread_fn)Async_addfile_thread, file);
+
+		queue_head = NULL;
+		queue_tail = NULL;
+		queue_size = 0;
+	}
+}

src/d_async.h

@@ -0,0 +1,44 @@
+// SONIC ROBO BLAST 2 KART
+//-----------------------------------------------------------------------------
+// Copyright (C) 2020 by James R.
+//
+// This program is free software distributed under the
+// terms of the GNU General Public License, version 2.
+// See the 'LICENSE' file for more details.
+//-----------------------------------------------------------------------------
+/// \file  d_async.h
+/// \brief Add wad/exec config asynchronously to the main thread, but in order
+
+#ifndef D_ASYNC_H
+#define D_ASYNC_H
+
+enum
+{
+	ASYNC_ADDFILE,
+	ASYNC_EXEC,
+};
+
+/*
+Queue a file to be added. Detach_async_addfile will spawn a thread that
+searches for any files if necessary.
+*/
+
+void Append_async_addfile (int type, const char * filename, void * ctx);
+
+/*
+Although the file searching is done on another thread, the actual adding of the
+wad or executing the config must be done on the main thread. This function is
+called every tic to check if a file was found so it can perform that last step.
+It does this in order.
+*/
+
+void Finish_async_addfile (void);
+
+/*
+Spawns a thread that searches for files that were queued.
+Called at the end of each tic.
+*/
+
+void Detach_async_addfile (void);
+
+#endif/*D_ASYNC_H*/

src/d_clisrv.c

@@ -46,6 +46,7 @@
 #include "lua_script.h"
 #include "lua_hook.h"
 #include "k_kart.h"
+#include "d_async.h"
 
 #ifdef CLIENT_LOADINGSCREEN
 // cl loading screen
@@ -5421,6 +5422,8 @@ void TryRunTics(tic_t realtics)
 	if (singletics)
 		realtics = 1;
 
+	Finish_async_addfile();
+
 	if (realtics >= 1)
 	{
 		COM_BufTicker();
@@ -5487,6 +5490,8 @@ void TryRunTics(tic_t realtics)
 	{
 		hu_stopped = true;
 	}
+
+	Detach_async_addfile();
 }
 
 

src/filesrch.c

@@ -462,7 +462,7 @@ filestatus_t filesearch(int nfiles, filequery_t *files, const char *startpath, b
 
 #ifdef HAVE_THREADS
 #define   Lock() I_lock_mutex    ((I_mutex *)mutex_ptr)
-#define Unlock() I_unlock_mutex (*(I_mutex *)mutex_ptr)
+#define Unlock() I_unlock_mutex (((mutex_ptr) ? *(I_mutex *)mutex_ptr : NULL))
 #else
 #define   Lock() ((void)0)
 #define Unlock() ((void)0)
@@ -600,6 +600,18 @@ filestatus_t filesearch(int nfiles, filequery_t *files, const char *startpath, b
 	free(searchpathindex);
 	free(dirhandle);
 
+	Lock();
+	{
+		for (i = 0; i < nfiles; ++i)
+		{
+			if (files[i].status != FS_FOUND && files[i].status != FS_MD5SUMBAD)
+			{
+				files[i].status = FS_NOTFOUND;
+			}
+		}
+	}
+	Unlock();
+
 	return ( (files_left) ? retval : FS_FOUND );
 
 #undef   Lock

src/m_misc.c

@@ -507,7 +507,7 @@ void M_FirstLoadConfig(void)
 	CV_InitFilterVar();
 
 	// load config, make sure those commands doesnt require the screen...
-	COM_BufInsertText(va("exec \"%s\" -singlethread\n", configfile));
+	COM_BufInsertText(va("exec \"%s\" -sync\n", configfile));
 	// no COM_BufExecute() needed; that does it right away
 
 	// don't filter anymore vars and don't let this convsvar be changed