added automatic process restarts for Linux dedicated server crashes and timed reboots

putting the UTC date and time in Linux crash report file names

changelog.txt

@@ -97,6 +97,10 @@ chg: reduced raw mouse input latency and added cl_drawMouseLag
 
 Linux:
 
+add: automatic dedicated server process restarts for crashes and timed reboots (sv_minRebootDelayMins)
+  this means 2 CNQ3 processes run per server: a parent (fixed pid) and a child (new pid after each restart)
+  this behavior can be disabled by passing "nohardreboot" as a command-line argument
+
 fix: tty handling of the leading "]" character
 
 fix: dedicated servers no longer insert a leading backslash when running auto-completion

code/qcommon/common.cpp

@@ -314,7 +314,7 @@ void QDECL Com_Error( int code, const char *fmt, ... )
 }
 
 
-void Com_Quit_f( void )
+void Com_Quit( int status )
 {
 	// don't try to shutdown if we are in a recursive error
 	if ( !com_errorEntered ) {
@@ -326,7 +326,13 @@ void Com_Quit_f( void )
 		FS_Shutdown( qtrue );
 	}
 
-	Sys_Quit();
+	Sys_Quit( status );
+}
+
+
+static void Com_Quit_f( void )
+{
+	Com_Quit( 0 );
 }
 
 

code/qcommon/qcommon.h

@@ -725,7 +725,7 @@ void		Info_Print( const char *s );
 void		Com_BeginRedirect (char *buffer, int buffersize, void (*flush)(char *));
 void		Com_EndRedirect( void );
 void		QDECL Com_DPrintf( const char *fmt, ... );
-void		Com_Quit_f( void );
+void		Com_Quit( int status );
 int			Com_EventLoop();
 int			Com_Milliseconds();	// will be journaled properly
 unsigned	Com_BlockChecksum( const void *buffer, int length );
@@ -962,7 +962,7 @@ typedef struct {
 sysEvent_t Sys_GetEvent();
 
 void Sys_Init();
-void Sys_Quit();
+void Sys_Quit( int status ); // status is the engine's exit code
 
 // general development dll loading for virtual machine testing
 void* QDECL Sys_LoadDll( const char* name, dllSyscall_t *entryPoint, dllSyscall_t systemcalls );
@@ -1001,6 +1001,8 @@ void	Sys_FreeFileList( char **list );
 
 qbool Sys_LowPhysicalMemory( void );
 
+qbool Sys_HardReboot(); // qtrue when the server can restart itself
+
 // huffman.cpp - id's original code
 // used for out-of-band (OOB) datagrams with dynamically created trees
 void	DynHuff_Compress( msg_t* buf, int offset );

code/server/sv_main.cpp

@@ -719,6 +719,9 @@ qbool SV_CheckPaused( void ) {
 
 static void SV_IntegerOverflowShutDown( const char* msg )
 {
+	if ( Sys_HardReboot() )
+		Com_Quit( 1 );
+
 	// save the map name in case it gets cleared during the shut down
 	char mapName[MAX_QPATH];
 	Q_strncpyz( mapName, Cvar_VariableString("mapname"), sizeof(mapName) );

code/unix/linux_local.h

@@ -40,5 +40,5 @@ void QGL_Shutdown( void );
 // signals.c
 void SIG_Init(void);
 
-extern int          q_argc;
-extern const char** q_argv;
+extern int    q_argc;
+extern char** q_argv;

code/unix/linux_signals.cpp

@@ -76,9 +76,25 @@ static const char* Sig_GetName(int sig)
 }
 
 
+static const char* Sig_CreateFileName(const char* extension)
+{
+	const time_t epochTime = time(NULL);
+	struct tm* const utcTime = gmtime(&epochTime);
+	const int y = 1900 + utcTime->tm_year;
+	const int m = utcTime->tm_mon + 1;
+	const int d = utcTime->tm_mday;
+	const int h = utcTime->tm_hour;
+	const int mi = utcTime->tm_min;
+	const int s = utcTime->tm_sec;
+
+	return va("%s-crash-%d.%02d.%02d-%02d.%02d.%02d%s",
+				q_argv[0], y, m, d, h, mi, s, extension);
+}
+
+
 static void Sig_WriteJSON(int sig)
 {
-	FILE* const file = fopen(va("%s-crash.json", q_argv[0]), "w");
+	FILE* const file = fopen(Sig_CreateFileName(".json"), "w");
 	if (file == NULL)
 		return;
 
@@ -95,7 +111,7 @@ static void Sig_WriteJSON(int sig)
 // only uses functions safe to call in a signal handler
 static void Sig_WriteBacktraceSafe()
 {
-	FILE* const file = fopen(va("%s-crash.bt", q_argv[0]), "w");
+	FILE* const file = fopen(Sig_CreateFileName(".bt"), "w");
 	if (file == NULL)
 		return;
 
@@ -125,7 +141,7 @@ static void libbt_ErrorCallback(void* data, const char* msg, int errnum)
 // might not be safe in a signal handler
 static void Sig_WriteBacktraceUnsafe()
 {
-	FILE* const file = fopen(va("%s-crash.bt", q_argv[0]), "a");
+	FILE* const file = fopen(Sig_CreateFileName(".bt"), "a");
 	if (file == NULL)
 		return;
 

code/unix/unix_main.cpp

@@ -53,8 +53,8 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 ///////////////////////////////////////////////////////////////
 
 
-int          q_argc = 0;
-const char** q_argv = NULL;
+int    q_argc = 0;
+char** q_argv = NULL;
 
 static qboolean stdin_active = qtrue;
 
@@ -521,10 +521,10 @@ void Sys_Error( const char *error, ... )
 }
 
 
-void Sys_Quit()
+void Sys_Quit( int status )
 {
 	Sys_ConsoleInputShutdown();
-	exit(0);
+	exit( status );
 }
 
 
@@ -837,11 +837,77 @@ void Sys_Init()
 }
 
 
-int main( int argc, const char** argv )
+qbool Sys_HardReboot()
+{
+#ifdef DEDICATED
+	return qtrue;
+#else
+	return qfalse;
+#endif
+}
+
+
+#ifdef DEDICATED
+
+
+static int Lin_RunProcess( char** argv )
+{
+	const pid_t pid = fork();
+	if (pid == 0) {
+		if (execve(argv[0], argv , NULL) == -1) {
+			fprintf(stderr, "failed to launch child process: %s\n", strerror(errno));
+			_exit(1); // quit without calling atexit handlers
+			return 0;
+		}
+	}
+
+	int status;
+	while (waitpid(pid, &status, WNOHANG) == 0)
+		sleep(1); // in seconds
+
+    return WEXITSTATUS(status);
+}
+
+
+static void Lin_HardRebootHandler( int argc, char** argv )
+{
+	for (int i = 0; i < argc; ++i) {
+		if (!Q_stricmp(argv[i], "nohardreboot")) {
+			return;
+		}
+	}
+	
+	static char* args[256];
+	if (argc + 2 >= sizeof(args) / sizeof(args[0])) {
+		fprintf(stderr, "too many arguments: %d\n", argc);
+		_exit(1); // quit without calling atexit handlers
+		return;
+	}
+
+	for (int i = 0; i < argc; ++i)
+		args[i] = argv[i];
+	args[argc + 0] = (char*)"nohardreboot";
+	args[argc + 1] = NULL;
+
+	for (;;) {
+		if (Lin_RunProcess(args) == 0)
+			_exit(0); // quit without calling atexit handlers
+	}
+}
+
+
+#endif
+
+
+int main( int argc, char** argv )
 {
 	q_argc = argc;
 	q_argv = argv;
 
+#ifdef DEDICATED
+	Lin_HardRebootHandler(argc, argv);
+#endif
+
 	SIG_Init();
 	
 	// merge the command line: we need it in a single chunk
@@ -863,7 +929,7 @@ int main( int argc, const char** argv )
 
 	Sys_ConsoleInputInit();
 
-	while (qtrue) {
+	for (;;) {
 		Com_Frame();
 	}
 

code/win32/win_main.cpp

@@ -98,7 +98,7 @@ void QDECL Sys_Error( const char *error, ... )
 	while (1) {
 		MSG msg;
 		if (!GetMessage(&msg, NULL, 0, 0))
-			Com_Quit_f();
+			Com_Quit( 1 );
 		TranslateMessage(&msg);
 		DispatchMessage(&msg);
 	}
@@ -109,14 +109,14 @@ void QDECL Sys_Error( const char *error, ... )
 }
 
 
-void Sys_Quit()
+void Sys_Quit( int status )
 {
 	WIN_EndTimePeriod();
 #ifndef DEDICATED
 	IN_Shutdown();
 #endif
 	Sys_DestroyConsole();
-	exit(0);
+	exit( status );
 }
 
 
@@ -467,7 +467,7 @@ sysEvent_t Sys_GetEvent()
 	MSG msg;
 	while (PeekMessage( &msg, NULL, 0, 0, PM_NOREMOVE )) {
 		if (!GetMessage( &msg, NULL, 0, 0 )) {
-			Com_Quit_f();
+			Com_Quit( 0 );
 		}
 
 		// save the msg time, because wndprocs don't have access to the timestamp

code/win32/win_shared.cpp

@@ -41,3 +41,9 @@ const char* Sys_DefaultHomePath()
 	return NULL;
 }
 
+
+qbool Sys_HardReboot()
+{
+	return qfalse;
+}
+