From b06f1b61283bcf29c38510c27db9bf168fb38605 Mon Sep 17 00:00:00 2001 From: Magnus Norddahl Date: Sun, 31 Oct 2021 03:05:43 +0100 Subject: [PATCH] Fix vulkan validation errors --- src/lightmap/gpuraytracer.cpp | 238 +++++++++++++++++++++++++++++++++- src/lightmap/gpuraytracer.h | 1 + src/lightmap/vulkandevice.cpp | 38 +++++- src/lightmap/vulkandevice.h | 3 +- 4 files changed, 267 insertions(+), 13 deletions(-) diff --git a/src/lightmap/gpuraytracer.cpp b/src/lightmap/gpuraytracer.cpp index 1848c8b..d424ad7 100644 --- a/src/lightmap/gpuraytracer.cpp +++ b/src/lightmap/gpuraytracer.cpp @@ -14,13 +14,232 @@ #include #include +#ifdef WIN32 +#include +#else +#include +#include +#include +#include +#include +#endif + extern int Multisample; extern int LightBounce; extern float GridSize; +#ifdef WIN32 +#pragma comment(lib, "dbghelp.lib") +class NativeSymbolResolver +{ +public: + NativeSymbolResolver() { SymInitialize(GetCurrentProcess(), nullptr, TRUE); } + ~NativeSymbolResolver() { SymCleanup(GetCurrentProcess()); } + + std::string GetName(void* frame) + { + std::string s; + + unsigned char buffer[sizeof(IMAGEHLP_SYMBOL64) + 128]; + IMAGEHLP_SYMBOL64* symbol64 = reinterpret_cast(buffer); + memset(symbol64, 0, sizeof(IMAGEHLP_SYMBOL64) + 128); + symbol64->SizeOfStruct = sizeof(IMAGEHLP_SYMBOL64); + symbol64->MaxNameLength = 128; + + DWORD64 displacement = 0; + BOOL result = SymGetSymFromAddr64(GetCurrentProcess(), (DWORD64)frame, &displacement, symbol64); + if (result) + { + IMAGEHLP_LINE64 line64; + DWORD displacement = 0; + memset(&line64, 0, sizeof(IMAGEHLP_LINE64)); + line64.SizeOfStruct = sizeof(IMAGEHLP_LINE64); + result = SymGetLineFromAddr64(GetCurrentProcess(), (DWORD64)frame, &displacement, &line64); + if (result) + { + s = std::string("Called from ") + symbol64->Name + " at " + line64.FileName + ", line " + std::to_string(line64.LineNumber) + "\n", symbol64->Name; + } + else + { + s = std::string("Called from ") + symbol64->Name + "\n"; + } + } + + return s; + } +}; +#else +class NativeSymbolResolver +{ +public: + std::string GetName(void* frame) + { + std::string s; + char** strings; + void* frames[1] = { frame }; + strings = backtrace_symbols(frames, 1); + + // Decode the strings + char* ptr = strings[0]; + char* filename = ptr; + const char* function = ""; + + // Find function name + while (*ptr) + { + if (*ptr == '(') // Found function name + { + *(ptr++) = 0; + function = ptr; + break; + } + ptr++; + } + + // Find offset + if (function[0]) // Only if function was found + { + while (*ptr) + { + if (*ptr == '+') // Found function offset + { + *(ptr++) = 0; + break; + } + if (*ptr == ')') // Not found function offset, but found, end of function + { + *(ptr++) = 0; + break; + } + ptr++; + } + } + + int status; + char* new_function = abi::__cxa_demangle(function, nullptr, nullptr, &status); + if (new_function) // Was correctly decoded + { + function = new_function; + } + + s = std::string("Called from ") + function + " at " + filename + "\n"; + + if (new_function) + { + free(new_function); + } + + free(strings); + return s; + } +}; +#endif + +static int CaptureStackTrace(int max_frames, void** out_frames) +{ + memset(out_frames, 0, sizeof(void*) * max_frames); + +#ifdef _WIN64 + // RtlCaptureStackBackTrace doesn't support RtlAddFunctionTable.. + + CONTEXT context; + RtlCaptureContext(&context); + + UNWIND_HISTORY_TABLE history; + memset(&history, 0, sizeof(UNWIND_HISTORY_TABLE)); + + ULONG64 establisherframe = 0; + PVOID handlerdata = nullptr; + + int frame; + for (frame = 0; frame < max_frames; frame++) + { + ULONG64 imagebase; + PRUNTIME_FUNCTION rtfunc = RtlLookupFunctionEntry(context.Rip, &imagebase, &history); + + KNONVOLATILE_CONTEXT_POINTERS nvcontext; + memset(&nvcontext, 0, sizeof(KNONVOLATILE_CONTEXT_POINTERS)); + if (!rtfunc) + { + // Leaf function + context.Rip = (ULONG64)(*(PULONG64)context.Rsp); + context.Rsp += 8; + } + else + { + RtlVirtualUnwind(UNW_FLAG_NHANDLER, imagebase, context.Rip, rtfunc, &context, &handlerdata, &establisherframe, &nvcontext); + } + + if (!context.Rip) + break; + + out_frames[frame] = (void*)context.Rip; + } + return frame; + +#elif defined(WIN32) + return 0;//return RtlCaptureStackBackTrace(0, MIN(max_frames, 32), out_frames, nullptr); +#else + return backtrace(out_frames, max_frames); +#endif +} + +std::string CaptureStackTraceText(int framesToSkip, bool includeNativeFrames) +{ + void* frames[32]; + int numframes = CaptureStackTrace(32, frames); + + std::unique_ptr nativeSymbols; + if (includeNativeFrames) + nativeSymbols.reset(new NativeSymbolResolver()); + + std::string s; + for (int i = framesToSkip + 1; i < numframes; i++) + { + s += nativeSymbols->GetName(frames[i]); + } + return s; +} + GPURaytracer::GPURaytracer() { - device = std::make_unique(); + auto printLog = [](const char* typestr, const std::string& msg) + { + printf("\n[%s] %s\n", typestr, msg.c_str()); + + std::string callstack = CaptureStackTraceText(0, true); + if (!callstack.empty()) + printf("%s\n", callstack.c_str()); + }; + device = std::make_unique(0, true, printLog); + + const auto& props = device->physicalDevice.properties; + + std::string deviceType; + switch (props.deviceType) + { + case VK_PHYSICAL_DEVICE_TYPE_OTHER: deviceType = "other"; break; + case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: deviceType = "integrated gpu"; break; + case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: deviceType = "discrete gpu"; break; + case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: deviceType = "virtual gpu"; break; + case VK_PHYSICAL_DEVICE_TYPE_CPU: deviceType = "cpu"; break; + default: deviceType = std::to_string(props.deviceType); break; + } + + std::string apiVersion = std::to_string(VK_VERSION_MAJOR(props.apiVersion)) + "." + std::to_string(VK_VERSION_MINOR(props.apiVersion)) + "." + std::to_string(VK_VERSION_PATCH(props.apiVersion)); + std::string driverVersion = std::to_string(VK_VERSION_MAJOR(props.driverVersion)) + "." + std::to_string(VK_VERSION_MINOR(props.driverVersion)) + "." + std::to_string(VK_VERSION_PATCH(props.driverVersion)); + + printf("Vulkan device: %s\n", props.deviceName); + printf("Vulkan device type: %s\n", deviceType.c_str()); + printf("Vulkan version: %s (api) %s (driver)\n", apiVersion.c_str(), driverVersion.c_str()); + /* + printf("Vulkan extensions:"); + for (const VkExtensionProperties& p : device->physicalDevice.extensions) + { + printf(" %s", p.extensionName); + } + printf("\n"); + */ } GPURaytracer::~GPURaytracer() @@ -33,6 +252,7 @@ void GPURaytracer::Raytrace(LevelMesh* level) cmdpool = std::make_unique(device.get(), device->graphicsFamily); cmdbuffer = cmdpool->createBuffer(); + cmdbuffer->begin(); printf("Creating vertex and index buffers\n"); CreateVertexAndIndexBuffers(); @@ -43,6 +263,8 @@ void GPURaytracer::Raytrace(LevelMesh* level) printf("Creating top level acceleration structure\n"); CreateTopLevelAccelerationStructure(); + cmdbuffer->end(); + printf("Tracing light probes\n"); Worker::RunJob((int)mesh->lightProbes.size(), [=](int id) { @@ -280,12 +502,12 @@ void GPURaytracer::CreateVertexAndIndexBuffers() size_t indexoffset = vertexoffset + vertexbuffersize; BufferBuilder vbuilder; - vbuilder.setUsage(VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + vbuilder.setUsage(VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT_KHR); vbuilder.setSize(vertexbuffersize); vertexBuffer = vbuilder.create(device.get()); BufferBuilder ibuilder; - ibuilder.setUsage(VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + ibuilder.setUsage(VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT_KHR); ibuilder.setSize(indexbuffersize); indexBuffer = ibuilder.create(device.get()); @@ -298,8 +520,8 @@ void GPURaytracer::CreateVertexAndIndexBuffers() memcpy(data + indexoffset, mesh->MeshElements.Data(), indexbuffersize); transferBuffer->Unmap(); - cmdbuffer->copyBuffer(transferBuffer.get(), vertexBuffer.get(), vertexoffset); - cmdbuffer->copyBuffer(transferBuffer.get(), indexBuffer.get(), indexoffset); + //cmdbuffer->copyBuffer(transferBuffer.get(), vertexBuffer.get(), vertexoffset); + //cmdbuffer->copyBuffer(transferBuffer.get(), indexBuffer.get(), indexoffset); VkMemoryBarrier barrier = { VK_STRUCTURE_TYPE_MEMORY_BARRIER }; barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; @@ -438,9 +660,15 @@ void GPURaytracer::CreateTopLevelAccelerationStructure() VkAccelerationStructureBuildSizesInfoKHR sizeInfo = { VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_SIZES_INFO_KHR }; vkGetAccelerationStructureBuildSizesKHR(device->device, VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, &buildInfo, &maxInstanceCount, &sizeInfo); + BufferBuilder tlbufbuilder; + tlbufbuilder.setUsage(VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + tlbufbuilder.setSize(sizeInfo.accelerationStructureSize); + tlAccelStructBuffer = tlbufbuilder.create(device.get()); + VkAccelerationStructureKHR tlAccelStructHandle = {}; VkAccelerationStructureCreateInfoKHR createInfo = { VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR }; createInfo.type = VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR; + createInfo.buffer = tlAccelStructBuffer->buffer; createInfo.size = sizeInfo.accelerationStructureSize; VkResult result = vkCreateAccelerationStructureKHR(device->device, &createInfo, nullptr, &tlAccelStructHandle); if (result != VK_SUCCESS) diff --git a/src/lightmap/gpuraytracer.h b/src/lightmap/gpuraytracer.h index bb28c1b..d4abccb 100644 --- a/src/lightmap/gpuraytracer.h +++ b/src/lightmap/gpuraytracer.h @@ -47,6 +47,7 @@ private: std::unique_ptr tlTransferBuffer; std::unique_ptr tlScratchBuffer; std::unique_ptr tlInstanceBuffer; + std::unique_ptr tlAccelStructBuffer; std::unique_ptr tlAccelStruct; std::unique_ptr cmdpool; diff --git a/src/lightmap/vulkandevice.cpp b/src/lightmap/vulkandevice.cpp index 2477f02..27e0dad 100644 --- a/src/lightmap/vulkandevice.cpp +++ b/src/lightmap/vulkandevice.cpp @@ -156,11 +156,14 @@ bool VulkanDevice::supportsDeviceExtension(const char *ext) const void VulkanDevice::createAllocator() { VmaAllocatorCreateInfo allocinfo = {}; + allocinfo.vulkanApiVersion = VK_API_VERSION_1_2; if (supportsDeviceExtension(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME) && supportsDeviceExtension(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME)) allocinfo.flags = VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT; + allocinfo.flags |= VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT; allocinfo.physicalDevice = physicalDevice.device; allocinfo.device = device; allocinfo.preferredLargeHeapBlockSize = 64 * 1024 * 1024; + allocinfo.instance = instance; if (vmaCreateAllocator(&allocinfo, &allocator) != VK_SUCCESS) throw std::runtime_error("Unable to create allocator"); } @@ -185,11 +188,28 @@ void VulkanDevice::createDevice() queueCreateInfos.push_back(queueCreateInfo); } + VkPhysicalDeviceRayTracingPipelineFeaturesKHR raytracingFeatures = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_RAY_TRACING_PIPELINE_FEATURES_KHR }; + raytracingFeatures.rayTracingPipeline = true; + raytracingFeatures.rayTraversalPrimitiveCulling = true; + + VkPhysicalDeviceAccelerationStructureFeaturesKHR deviceAccelFeatures = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ACCELERATION_STRUCTURE_FEATURES_KHR }; + deviceAccelFeatures.accelerationStructure = true; + deviceAccelFeatures.pNext = &raytracingFeatures; + + VkPhysicalDeviceBufferDeviceAddressFeatures deviceAddressFeatures = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES }; + deviceAddressFeatures.bufferDeviceAddress = true; + deviceAddressFeatures.pNext = &deviceAccelFeatures; + + VkPhysicalDeviceFeatures2 deviceFeatures2 = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2 }; + deviceFeatures2.features = enabledDeviceFeatures; + deviceFeatures2.pNext = &deviceAddressFeatures; + VkDeviceCreateInfo deviceCreateInfo = {}; deviceCreateInfo.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO; + deviceCreateInfo.pNext = &deviceFeatures2; deviceCreateInfo.queueCreateInfoCount = (uint32_t)queueCreateInfos.size(); deviceCreateInfo.pQueueCreateInfos = queueCreateInfos.data(); - deviceCreateInfo.pEnabledFeatures = &enabledDeviceFeatures; + //deviceCreateInfo.pEnabledFeatures = &enabledDeviceFeatures; deviceCreateInfo.enabledExtensionCount = (uint32_t)enabledDeviceExtensions.size(); deviceCreateInfo.ppEnabledExtensionNames = enabledDeviceExtensions.data(); deviceCreateInfo.enabledLayerCount = 0; @@ -255,16 +275,20 @@ void VulkanDevice::createInstance() extensions = getExtensions(); enabledExtensions = getPlatformExtensions(); - std::string debugLayer = "VK_LAYER_LUNARG_standard_validation"; + std::string debugLayer = "VK_LAYER_KHRONOS_validation"; bool wantDebugLayer = vk_debug; bool debugLayerFound = false; - for (const VkLayerProperties &layer : availableLayers) + if (wantDebugLayer) { - if (layer.layerName == debugLayer && wantDebugLayer) + for (const VkLayerProperties& layer : availableLayers) { - enabledValidationLayers.push_back(debugLayer.c_str()); - enabledExtensions.push_back(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); - debugLayerFound = true; + if (layer.layerName == debugLayer) + { + enabledValidationLayers.push_back(layer.layerName); + enabledExtensions.push_back(VK_EXT_DEBUG_UTILS_EXTENSION_NAME); + debugLayerFound = true; + break; + } } } diff --git a/src/lightmap/vulkandevice.h b/src/lightmap/vulkandevice.h index 964b3ea..e14a9bf 100644 --- a/src/lightmap/vulkandevice.h +++ b/src/lightmap/vulkandevice.h @@ -75,7 +75,8 @@ public: //VK_KHR_SWAPCHAIN_EXTENSION_NAME, VK_KHR_RAY_QUERY_EXTENSION_NAME, VK_KHR_RAY_TRACING_PIPELINE_EXTENSION_NAME, - VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME + VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME, + VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME }; std::vector optionalDeviceExtensions = { //VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME,