- enable console subsystem processing for Windows, enable runtime detection

This commit is contained in:
Rachael Alexanderson 2024-10-15 10:50:04 -04:00
parent ee5442c06b
commit 70eb4b974d
No known key found for this signature in database
GPG key ID: 26A8ACCE97115EE0
2 changed files with 47 additions and 1 deletions

View file

@ -1359,6 +1359,13 @@ else()
endif()
if( MSVC )
option ( CONSOLE_MODE "Compile as a console application" OFF )
if ( CONSOLE_MODE )
set ( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SUBSYSTEM:CONSOLE /ENTRY:wWinMainCRTStartup" )
else()
set ( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SUBSYSTEM:WINDOWS /ENTRY:wWinMainCRTStartup" )
endif()
option( ZDOOM_GENERATE_MAPFILE "Generate .map file for debugging." OFF )
set( LINKERSTUFF "/MANIFEST:NO" )

View file

@ -134,6 +134,28 @@ void I_SetIWADInfo()
{
}
//==========================================================================
//
// isConsoleApp()
//
// runtime detection to detect if this is a console subsystem app.
//
// the reason for doing this is because it is possible to edit a binary directly and change its subsystem
// type via hexedit so in order to gain flexibility it makes no sense to just compile out the unused code.
//
// we may plan to publish tools to allow users to do this manually on their own.
//
//==========================================================================
bool isConsoleApp()
{
DWORD pids[2];
DWORD num_pids = GetConsoleProcessList(pids, 2);
bool win32con_is_exclusive = (num_pids <= 1);
return GetConsoleWindow() != NULL && !win32con_is_exclusive;
}
//==========================================================================
//
// DoMain
@ -158,7 +180,24 @@ int DoMain (HINSTANCE hInstance)
Args->AppendArg(FString(wargv[i]));
}
if (Args->CheckParm("-stdout") || Args->CheckParm("-norun"))
if (isConsoleApp())
{
StdOut = GetStdHandle(STD_OUTPUT_HANDLE);
BY_HANDLE_FILE_INFORMATION info;
if (!GetFileInformationByHandle(StdOut, &info) && StdOut != nullptr)
{
SetConsoleCP(CP_UTF8);
SetConsoleOutputCP(CP_UTF8);
DWORD mode;
if (GetConsoleMode(StdOut, &mode))
{
if (SetConsoleMode(StdOut, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING))
FancyStdOut = IsWindows10OrGreater(); // Windows 8.1 and lower do not understand ANSI formatting.
}
}
}
else if (Args->CheckParm("-stdout") || Args->CheckParm("-norun"))
{
// As a GUI application, we don't normally get a console when we start.
// If we were run from the shell and are on XP+, we can attach to its