Improve crash handling in drawers

src/r_compiler/llvmdrawers.h

@@ -41,6 +41,13 @@ struct DrawWallArgs
 		simple_shade = 1,
 		nearest_filter = 2
 	};
+
+	FString ToString()
+	{
+		FString info;
+		info.Format("dest_y = %i, count = %i, flags = %i", dest_y, count, flags);
+		return info;
+	}
 };
 
 struct DrawSpanArgs

src/r_draw_rgba.cpp

@@ -104,6 +104,8 @@ public:
 		LLVMDrawers::Instance()->DrawSpan(&args);
 	}
 
+	FString DebugInfo() override { return "DrawSpanLLVMCommand"; }
+
 protected:
 	DrawSpanArgs args;
 
@@ -247,6 +249,11 @@ public:
 		WorkerThreadData d = ThreadData(thread);
 		LLVMDrawers::Instance()->vlinec4(&args, &d);
 	}
+
+	FString DebugInfo() override
+	{
+		return "DrawWall4LLVMCommand\n" + args.ToString();
+	}
 };
 
 class DrawWall1LLVMCommand : public DrawerCommand
@@ -301,6 +308,11 @@ public:
 		WorkerThreadData d = ThreadData(thread);
 		LLVMDrawers::Instance()->vlinec1(&args, &d);
 	}
+
+	FString DebugInfo() override
+	{
+		return "DrawWall1LLVMCommand\n" + args.ToString();
+	}
 };
 
 class DrawColumnLLVMCommand : public DrawerCommand
@@ -318,6 +330,11 @@ protected:
 		return d;
 	}
 
+	FString DebugInfo() override
+	{
+		return "DrawColumnLLVMCommand";
+	}
+
 public:
 	DrawColumnLLVMCommand()
 	{
@@ -510,6 +527,11 @@ public:
 			*dest = 0xff000000 | (red << 16) | (green << 8) | blue;
 		}
 	}
+
+	FString DebugInfo() override
+	{
+		return "DrawFuzzColumnRGBACommand";
+	}
 };
 
 class FillSpanRGBACommand : public DrawerCommand
@@ -544,6 +566,11 @@ public:
 		for (int i = 0; i < count; i++)
 			dest[i] = color;
 	}
+
+	FString DebugInfo() override
+	{
+		return "FillSpanRGBACommand";
+	}
 };
 
 /////////////////////////////////////////////////////////////////////////////
@@ -663,6 +690,11 @@ public:
 			dy--;
 		}
 	}
+
+	FString DebugInfo() override
+	{
+		return "DrawSlabRGBACommand";
+	}
 };
 
 /////////////////////////////////////////////////////////////////////////////
@@ -737,6 +769,11 @@ public:
 			dest[x] = 0xff000000 | (red << 16) | (green << 8) | blue;
 		} while (++x <= x2);
 	}
+
+	FString DebugInfo() override
+	{
+		return "DrawFogBoundaryLineRGBACommand";
+	}
 };
 
 class DrawTiltedSpanRGBACommand : public DrawerCommand
@@ -886,6 +923,11 @@ public:
 			count--;
 		}
 	}
+
+	FString DebugInfo() override
+	{
+		return "DrawTiltedSpanRGBACommand";
+	}
 };
 
 class DrawColoredSpanRGBACommand : public DrawerCommand
@@ -925,6 +967,11 @@ public:
 		for (int i = 0; i < count; i++)
 			dest[i] = color;
 	}
+
+	FString DebugInfo() override
+	{
+		return "DrawColoredSpanRGBACommand";
+	}
 };
 
 class FillTransColumnRGBACommand : public DrawerCommand
@@ -992,6 +1039,11 @@ public:
 			dest += spacing;
 		}
 	}
+
+	FString DebugInfo() override
+	{
+		return "FillTransColumnRGBACommand";
+	}
 };
 
 ApplySpecialColormapRGBACommand::ApplySpecialColormapRGBACommand(FSpecialColormap *colormap, DFrameBuffer *screen)

src/r_draw_rgba.h

@@ -137,6 +137,7 @@ class ApplySpecialColormapRGBACommand : public DrawerCommand
 public:
 	ApplySpecialColormapRGBACommand(FSpecialColormap *colormap, DFrameBuffer *screen);
 	void Execute(DrawerThread *thread) override;
+	FString DebugInfo() override { return "ApplySpecialColormapRGBACommand"; }
 };
 
 template<typename CommandType, typename BlendMode>

src/r_drawt_rgba.cpp

@@ -104,6 +104,11 @@ public:
 		WorkerThreadData d = ThreadData(thread);
 		LLVMDrawers::Instance()->DrawColumnRt1(&args, &d);
 	}
+
+	FString DebugInfo() override
+	{
+		return "DrawColumnRt1LLVMCommand";
+	}
 };
 
 #define DECLARE_DRAW_COMMAND(name, func, base) \
@@ -158,6 +163,11 @@ public:
 	{
 		thread->dc_temp_rgba = buff == NULL ? thread->dc_temp_rgbabuff_rgba : (uint32_t*)buff;
 	}
+
+	FString DebugInfo() override
+	{
+		return "RtInitColsRGBACommand";
+	}
 };
 
 template<typename InputPixelType>
@@ -233,6 +243,11 @@ public:
 			dest += 32;
 		} while (--count);
 	}
+
+	FString DebugInfo() override
+	{
+		return "DrawColumnHorizRGBACommand";
+	}
 };
 
 class FillColumnHorizRGBACommand : public DrawerCommand
@@ -278,6 +293,11 @@ public:
 			dest += 8;
 		} while (--count);
 	}
+
+	FString DebugInfo() override
+	{
+		return "FillColumnHorizRGBACommand";
+	}
 };
 
 /////////////////////////////////////////////////////////////////////////////

src/r_thread.cpp

@@ -97,26 +97,50 @@ void DrawerCommandQueue::Finish()
 	thread.core = 0;
 	thread.num_cores = (int)(queue->threads.size() + 1);
 
-	for (int pass = 0; pass < queue->num_passes; pass++)
+	struct TryCatchData
 	{
-		thread.pass_start_y = pass * queue->rows_in_pass;
+		DrawerCommandQueue *queue;
-		thread.pass_end_y = (pass + 1) * queue->rows_in_pass;
+		DrawerThread *thread;
-		if (pass + 1 == queue->num_passes)
+		size_t command_index;
-			thread.pass_end_y = MAX(thread.pass_end_y, MAXHEIGHT);
+	} data;
 
-		size_t size = queue->active_commands.size();
+	data.queue = queue;
-		for (size_t i = 0; i < size; i++)
+	data.thread = &thread;
+	data.command_index = 0;
+	VectoredTryCatch(&data,
+	[](void *data)
+	{
+		TryCatchData *d = (TryCatchData*)data;
+
+		for (int pass = 0; pass < d->queue->num_passes; pass++)
 		{
-			auto &command = queue->active_commands[i];
+			d->thread->pass_start_y = pass * d->queue->rows_in_pass;
-			command->Execute(&thread);
+			d->thread->pass_end_y = (pass + 1) * d->queue->rows_in_pass;
+			if (pass + 1 == d->queue->num_passes)
+				d->thread->pass_end_y = MAX(d->thread->pass_end_y, MAXHEIGHT);
+
+			size_t size = d->queue->active_commands.size();
+			for (d->command_index = 0; d->command_index < size; d->command_index++)
+			{
+				auto &command = d->queue->active_commands[d->command_index];
+				command->Execute(d->thread);
+			}
 		}
-	}
+	},
+	[](void *data)
+	{
+		TryCatchData *d = (TryCatchData*)data;
+		ReportFatalError(d->queue->active_commands[d->command_index], true);
+	});
 
 	// Wait for everyone to finish:
 
 	std::unique_lock<std::mutex> end_lock(queue->end_mutex);
 	queue->end_condition.wait(end_lock, [&]() { return queue->finished_threads == queue->threads.size(); });
 
+	if (!queue->thread_error.IsEmpty())
+		I_FatalError("Fatal drawer error: %s", queue->thread_error.GetChars());
+
 	// Clean up batch:
 
 	for (auto &command : queue->active_commands)
@@ -157,20 +181,42 @@ void DrawerCommandQueue::StartThreads()
 				start_lock.unlock();
 
 				// Do the work:
-				for (int pass = 0; pass < queue->num_passes; pass++)
-				{
-					thread->pass_start_y = pass * queue->rows_in_pass;
-					thread->pass_end_y = (pass + 1) * queue->rows_in_pass;
-					if (pass + 1 == queue->num_passes)
-						thread->pass_end_y = MAX(thread->pass_end_y, MAXHEIGHT);
 
-					size_t size = queue->active_commands.size();
+				struct TryCatchData
-					for (size_t i = 0; i < size; i++)
+				{
+					DrawerCommandQueue *queue;
+					DrawerThread *thread;
+					size_t command_index;
+				} data;
+
+				data.queue = queue;
+				data.thread = thread;
+				data.command_index = 0;
+				VectoredTryCatch(&data,
+				[](void *data)
+				{
+					TryCatchData *d = (TryCatchData*)data;
+
+					for (int pass = 0; pass < d->queue->num_passes; pass++)
 					{
-						auto &command = queue->active_commands[i];
+						d->thread->pass_start_y = pass * d->queue->rows_in_pass;
-						command->Execute(thread);
+						d->thread->pass_end_y = (pass + 1) * d->queue->rows_in_pass;
+						if (pass + 1 == d->queue->num_passes)
+							d->thread->pass_end_y = MAX(d->thread->pass_end_y, MAXHEIGHT);
+
+						size_t size = d->queue->active_commands.size();
+						for (d->command_index = 0; d->command_index < size; d->command_index++)
+						{
+							auto &command = d->queue->active_commands[d->command_index];
+							command->Execute(d->thread);
+						}
 					}
-				}
+				},
+				[](void *data)
+				{
+					TryCatchData *d = (TryCatchData*)data;
+					ReportFatalError(d->queue->active_commands[d->command_index], true);
+				});
 
 				// Notify main thread that we finished:
 				std::unique_lock<std::mutex> end_lock(queue->end_mutex);
@@ -194,3 +240,26 @@ void DrawerCommandQueue::StopThreads()
 	lock.lock();
 	shutdown_flag = false;
 }
+
+void DrawerCommandQueue::ReportFatalError(DrawerCommand *command, bool worker_thread)
+{
+	if (worker_thread)
+	{
+		std::unique_lock<std::mutex> end_lock(Instance()->end_mutex);
+		if (Instance()->thread_error.IsEmpty())
+			Instance()->thread_error = command->DebugInfo();
+	}
+	else
+	{
+		I_FatalError("Fatal drawer error: %s", command->DebugInfo().GetChars());
+	}
+}
+
+#ifndef WIN32
+
+void VectoredTryCatch(void *data, void(*tryBlock)(void *data), void(*catchBlock)(void *data))
+{
+	tryBlock(data);
+}
+
+#endif

src/r_thread.h

@@ -79,8 +79,11 @@ public:
 	}
 
 	virtual void Execute(DrawerThread *thread) = 0;
+	virtual FString DebugInfo() = 0;
 };
 
+void VectoredTryCatch(void *data, void(*tryBlock)(void *data), void(*catchBlock)(void *data));
+
 // Manages queueing up commands and executing them on worker threads
 class DrawerCommandQueue
 {
@@ -101,6 +104,7 @@ class DrawerCommandQueue
 	std::mutex end_mutex;
 	std::condition_variable end_condition;
 	size_t finished_threads = 0;
+	FString thread_error;
 
 	int threaded_render = 0;
 	DrawerThread single_core_thread;
@@ -112,6 +116,7 @@ class DrawerCommandQueue
 	void Finish();
 
 	static DrawerCommandQueue *Instance();
+	static void ReportFatalError(DrawerCommand *command, bool worker_thread);
 
 	DrawerCommandQueue();
 	~DrawerCommandQueue();
@@ -128,7 +133,17 @@ public:
 		if (queue->threaded_render == 0 || !r_multithreaded)
 		{
 			T command(std::forward<Types>(args)...);
-			command.Execute(&queue->single_core_thread);
+			VectoredTryCatch(&command,
+			[](void *data)
+			{
+				T *c = (T*)data;
+				c->Execute(&Instance()->single_core_thread);
+			},
+			[](void *data)
+			{
+				T *c = (T*)data;
+				ReportFatalError(c, false);
+			});
 		}
 		else
 		{

src/win32/i_crash.cpp

@@ -3399,3 +3399,37 @@ void DisplayCrashLog ()
 	}
 	CloseTarFiles ();
 }
+
+/////////////////////////////////////////////////////////////////////////////
+
+namespace
+{
+	bool __declspec(thread) DrawerExceptionSetJumpResult;
+	CONTEXT __declspec(thread) DrawerExceptionSetJumpContext;
+	PVOID __declspec(thread) DrawerExceptionHandlerHandle;
+
+	LONG WINAPI DrawerExceptionHandler(_EXCEPTION_POINTERS *exceptionInfo)
+	{
+		//RtlRestoreContext(&DrawerExceptionSetJumpContext, exceptionInfo->ExceptionRecord);
+		*exceptionInfo->ContextRecord = DrawerExceptionSetJumpContext;
+		return EXCEPTION_CONTINUE_EXECUTION;
+	}
+}
+
+void VectoredTryCatch(void *data, void(*tryBlock)(void *data), void(*catchBlock)(void *data))
+{
+	DrawerExceptionSetJumpResult = false;
+	RtlCaptureContext(&DrawerExceptionSetJumpContext);
+	if (DrawerExceptionSetJumpResult)
+	{
+		RemoveVectoredExceptionHandler(DrawerExceptionHandlerHandle);
+		catchBlock(data);
+	}
+	else
+	{
+		DrawerExceptionSetJumpResult = true;
+		DrawerExceptionHandlerHandle = AddVectoredExceptionHandler(1, DrawerExceptionHandler);
+		tryBlock(data);
+		RemoveVectoredExceptionHandler(DrawerExceptionHandlerHandle);
+	}
+}