From 70eb4b974dfc7381d1b72e7c66acdd83f69f7359 Mon Sep 17 00:00:00 2001
From: Rachael Alexanderson <madame-rachelle@users.noreply.github.com>
Date: Tue, 15 Oct 2024 10:50:04 -0400
Subject: [PATCH] - enable console subsystem processing for Windows, enable
 runtime detection

---
 src/CMakeLists.txt                   |  7 +++++
 src/common/platform/win32/i_main.cpp | 41 +++++++++++++++++++++++++++-
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 789bea954f..987f53dc0f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1359,6 +1359,13 @@ else()
 endif()
 
 if( MSVC )
+	option ( CONSOLE_MODE "Compile as a console application" OFF )
+	if ( CONSOLE_MODE )
+		set ( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SUBSYSTEM:CONSOLE /ENTRY:wWinMainCRTStartup" )
+	else()
+		set ( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SUBSYSTEM:WINDOWS /ENTRY:wWinMainCRTStartup" )
+	endif()
+
 	option( ZDOOM_GENERATE_MAPFILE "Generate .map file for debugging." OFF )
 	set( LINKERSTUFF "/MANIFEST:NO" )
 
diff --git a/src/common/platform/win32/i_main.cpp b/src/common/platform/win32/i_main.cpp
index acadc2dc82..64c74e8080 100644
--- a/src/common/platform/win32/i_main.cpp
+++ b/src/common/platform/win32/i_main.cpp
@@ -134,6 +134,28 @@ void I_SetIWADInfo()
 {
 }
 
+//==========================================================================
+//
+// isConsoleApp()
+//
+// runtime detection to detect if this is a console subsystem app.
+//
+// the reason for doing this is because it is possible to edit a binary directly and change its subsystem
+// type via hexedit so in order to gain flexibility it makes no sense to just compile out the unused code.
+//
+// we may plan to publish tools to allow users to do this manually on their own.
+//
+//==========================================================================
+
+bool isConsoleApp()
+{
+	DWORD pids[2];
+	DWORD num_pids = GetConsoleProcessList(pids, 2);
+	bool win32con_is_exclusive = (num_pids <= 1);
+
+	return GetConsoleWindow() != NULL && !win32con_is_exclusive;
+}
+
 //==========================================================================
 //
 // DoMain
@@ -158,7 +180,24 @@ int DoMain (HINSTANCE hInstance)
 		Args->AppendArg(FString(wargv[i]));
 	}
 
-	if (Args->CheckParm("-stdout") || Args->CheckParm("-norun"))
+	if (isConsoleApp())
+	{
+		StdOut = GetStdHandle(STD_OUTPUT_HANDLE);
+		BY_HANDLE_FILE_INFORMATION info;
+
+		if (!GetFileInformationByHandle(StdOut, &info) && StdOut != nullptr)
+		{
+			SetConsoleCP(CP_UTF8);
+			SetConsoleOutputCP(CP_UTF8);
+			DWORD mode;
+			if (GetConsoleMode(StdOut, &mode))
+			{
+				if (SetConsoleMode(StdOut, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING))
+					FancyStdOut = IsWindows10OrGreater(); // Windows 8.1 and lower do not understand ANSI formatting.
+			}
+		}
+	}
+	else if (Args->CheckParm("-stdout") || Args->CheckParm("-norun"))
 	{
 		// As a GUI application, we don't normally get a console when we start.
 		// If we were run from the shell and are on XP+, we can attach to its