diff --git a/CMakeLists.txt b/CMakeLists.txt
index 006ebf476f..22973e19ae 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -188,6 +188,16 @@ option(FORCE_INTERNAL_JPEG "Use internal jpeg")
 option(FORCE_INTERNAL_BZIP2 "Use internal bzip2")
 option(FORCE_INTERNAL_GME "Use internal gme" ON)
 
+# Fast math flags, required by some subprojects
+set( ZD_FASTMATH_FLAG "" )
+if( ZD_CMAKE_COMPILER_IS_GNUCXX_COMPATIBLE )
+	set( ZD_FASTMATH_FLAG "-ffast-math" )
+else()
+	if( MSVC )
+		set( ZD_FASTMATH_FLAG "/fp:fast" )
+	endif()
+endif()
+
 if( ZLIB_FOUND AND NOT FORCE_INTERNAL_ZLIB )
 	message( STATUS "Using system zlib" )
 else()
diff --git a/dumb/CMakeLists.txt b/dumb/CMakeLists.txt
index c8fa34bf7b..8455e6d15c 100644
--- a/dumb/CMakeLists.txt
+++ b/dumb/CMakeLists.txt
@@ -14,9 +14,8 @@ if( ZD_CMAKE_COMPILER_IS_GNUC_COMPATIBLE )
 	endif(NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "4.5")
 endif()
 
-if( MSVC )
-	set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /fp:fast" )
-endif()
+# Enable fast flag for dumb
+set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ZD_FASTMATH_FLAG}" )
 
 CHECK_FUNCTION_EXISTS( itoa ITOA_EXISTS )
 if( NOT ITOA_EXISTS )
diff --git a/game-music-emu/CMakeLists.txt b/game-music-emu/CMakeLists.txt
index f1d467bcc0..3251b5c02c 100644
--- a/game-music-emu/CMakeLists.txt
+++ b/game-music-emu/CMakeLists.txt
@@ -29,11 +29,8 @@ if( ZD_CMAKE_COMPILER_IS_GNUCXX_COMPATIBLE )
 	endif()
 endif()
 
-if( MSVC )
-	set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /fp:fast" )
-	set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /fp:fast" )
-endif()
-
+# Enable fast flag for GME
+set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ZD_FASTMATH_FLAG}" )
 
 # Default emulators to build (all of them! ;)
 if (NOT DEFINED USE_GME_AY)
diff --git a/gdtoa/CMakeLists.txt b/gdtoa/CMakeLists.txt
index 94ab9a6914..c165422834 100644
--- a/gdtoa/CMakeLists.txt
+++ b/gdtoa/CMakeLists.txt
@@ -5,13 +5,16 @@ set( CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -D_DEBUG" )
 # Disable warnings for << operator precedence (4554) and
 # unreferenced labels (4102) from VC
 if( MSVC )
-	set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4554 /wd4102 /fp:fast" )
+	set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4554 /wd4102" )
 endif()
 
 if( ZD_CMAKE_COMPILER_IS_GNUCXX_COMPATIBLE )
 	set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra" )
 endif()
 
+# Enable fast flag for gdtoa
+set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ZD_FASTMATH_FLAG}" )
+
 include_directories( ${CMAKE_CURRENT_BINARY_DIR} )
 add_definitions( -DINFNAN_CHECK -DMULTIPLE_THREADS )
 
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 6e617c7123..21fb48eec5 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -35,14 +35,6 @@ if( CMAKE_SIZEOF_VOID_P MATCHES "8" )
 	set( X64 64 )
 endif()
 
-if( ZD_CMAKE_COMPILER_IS_GNUCXX_COMPATIBLE )
-	set ( FASTMATH "-ffast-math" )
-else()
-	if ( MSVC )
-		set ( FASTMATH "/fp:fast" )
-	endif()
-endif()
-
 # You can either use "make install" on the FMOD distribution to put it
 # in standard system locations, or you can unpack the FMOD distribution
 # in the root of the zdoom tree. e.g.:
@@ -866,6 +858,7 @@ set( NOT_COMPILED_SOURCE_FILES
 	asm_x86_64/tmap3.s
 )
 
+# Enable fast math for some sources
 set( FASTMATH_SOURCES
 	r_swrenderer.cpp
 	r_3dfloors.cpp
@@ -1203,12 +1196,12 @@ add_executable( zdoom WIN32 MACOSX_BUNDLE
 	zzautozend.cpp
 )
 
-set_source_files_properties( ${FASTMATH_SOURCES} PROPERTIES COMPILE_FLAGS ${FASTMATH} )
+set_source_files_properties( ${FASTMATH_SOURCES} PROPERTIES COMPILE_FLAGS ${ZD_FASTMATH_FLAG} )
 set_source_files_properties( xlat/parse_xlat.cpp PROPERTIES OBJECT_DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/xlat_parser.c" )
 set_source_files_properties( sc_man.cpp PROPERTIES OBJECT_DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/sc_man_scanner.h" )
 set_source_files_properties( ${NOT_COMPILED_SOURCE_FILES} PROPERTIES HEADER_FILE_ONLY TRUE )
 if ( WIN32 )
-	set_source_files_properties( win32/fb_d3d9.cpp win32/fb_d3d9_wipe.cpp PROPERTIES COMPILE_FLAGS ${FASTMATH} )
+	set_source_files_properties( win32/fb_d3d9.cpp win32/fb_d3d9_wipe.cpp PROPERTIES COMPILE_FLAGS ${ZD_FASTMATH_FLAG} )
 endif()