From 67e63653a4851c201ea92d5601b2285b6ec49162 Mon Sep 17 00:00:00 2001 From: Kelsi Date: Sun, 22 Feb 2026 10:23:20 -0800 Subject: [PATCH] Stabilize Vulkan shadow pipeline diagnostics and compatibility path - Fix shadow depth image layout transitions by tracking per-frame old/new layouts. - Update receiver shadow projection to Vulkan clip-depth convention. - Test inverted shadow compare op path (GREATER_OR_EQUAL). - Switch shadow compare samplers to NEAREST filtering for broader Vulkan compatibility. - Expand shadow caster coverage by disabling caster cull filtering in WMO/M2/Character shadow pipelines. - Keep light-space matrix path on stable character-centered framing. --- assets/shaders/character.frag.glsl | 9 ++++++--- assets/shaders/m2.frag.glsl | 9 ++++++--- assets/shaders/terrain.frag.glsl | 2 +- assets/shaders/wmo.frag.glsl | 9 ++++++--- include/rendering/renderer.hpp | 1 + src/rendering/character_renderer.cpp | 4 ++-- src/rendering/m2_renderer.cpp | 11 +++++++---- src/rendering/renderer.cpp | 27 ++++++++++++++++++--------- src/rendering/vk_texture.cpp | 6 +++--- src/rendering/wmo_renderer.cpp | 9 +++++---- 10 files changed, 55 insertions(+), 32 deletions(-) diff --git a/assets/shaders/character.frag.glsl b/assets/shaders/character.frag.glsl index 569c099d..758c7d2a 100644 --- a/assets/shaders/character.frag.glsl +++ b/assets/shaders/character.frag.glsl @@ -63,9 +63,12 @@ void main() { float shadow = 1.0; if (shadowParams.x > 0.5) { vec4 lsPos = lightSpaceMatrix * vec4(FragPos, 1.0); - vec3 proj = lsPos.xyz / lsPos.w * 0.5 + 0.5; - if (proj.z <= 1.0) { - float bias = max(0.005 * (1.0 - dot(norm, ldir)), 0.001); + vec3 proj = lsPos.xyz / lsPos.w; + proj.xy = proj.xy * 0.5 + 0.5; + if (proj.x >= 0.0 && proj.x <= 1.0 && + proj.y >= 0.0 && proj.y <= 1.0 && + proj.z >= 0.0 && proj.z <= 1.0) { + float bias = max(0.0005 * (1.0 - dot(norm, ldir)), 0.00005); shadow = texture(uShadowMap, vec3(proj.xy, proj.z - bias)); } shadow = mix(1.0, shadow, shadowParams.y); diff --git a/assets/shaders/m2.frag.glsl b/assets/shaders/m2.frag.glsl index bdc8e0e9..bd91744f 100644 --- a/assets/shaders/m2.frag.glsl +++ b/assets/shaders/m2.frag.glsl @@ -86,9 +86,12 @@ void main() { float shadow = 1.0; if (shadowParams.x > 0.5) { vec4 lsPos = lightSpaceMatrix * vec4(FragPos, 1.0); - vec3 proj = lsPos.xyz / lsPos.w * 0.5 + 0.5; - if (proj.z <= 1.0) { - float bias = max(0.005 * (1.0 - abs(dot(norm, ldir))), 0.001); + vec3 proj = lsPos.xyz / lsPos.w; + proj.xy = proj.xy * 0.5 + 0.5; + if (proj.x >= 0.0 && proj.x <= 1.0 && + proj.y >= 0.0 && proj.y <= 1.0 && + proj.z >= 0.0 && proj.z <= 1.0) { + float bias = max(0.0005 * (1.0 - abs(dot(norm, ldir))), 0.00005); shadow = texture(uShadowMap, vec3(proj.xy, proj.z - bias)); } shadow = mix(1.0, shadow, shadowParams.y); diff --git a/assets/shaders/terrain.frag.glsl b/assets/shaders/terrain.frag.glsl index be0bfc98..21d51e38 100644 --- a/assets/shaders/terrain.frag.glsl +++ b/assets/shaders/terrain.frag.glsl @@ -84,7 +84,7 @@ void main() { vec3 proj = lsPos.xyz / lsPos.w; proj.xy = proj.xy * 0.5 + 0.5; if (proj.x >= 0.0 && proj.x <= 1.0 && proj.y >= 0.0 && proj.y <= 1.0 && proj.z <= 1.0) { - float bias = 0.002; + float bias = 0.0002; shadow = texture(uShadowMap, vec3(proj.xy, proj.z - bias)); shadow = mix(1.0, shadow, shadowParams.y); } diff --git a/assets/shaders/wmo.frag.glsl b/assets/shaders/wmo.frag.glsl index faf44858..3a5c0f21 100644 --- a/assets/shaders/wmo.frag.glsl +++ b/assets/shaders/wmo.frag.glsl @@ -57,9 +57,12 @@ void main() { float shadow = 1.0; if (shadowParams.x > 0.5) { vec4 lsPos = lightSpaceMatrix * vec4(FragPos, 1.0); - vec3 proj = lsPos.xyz / lsPos.w * 0.5 + 0.5; - if (proj.z <= 1.0) { - float bias = max(0.005 * (1.0 - dot(norm, ldir)), 0.001); + vec3 proj = lsPos.xyz / lsPos.w; + proj.xy = proj.xy * 0.5 + 0.5; + if (proj.x >= 0.0 && proj.x <= 1.0 && + proj.y >= 0.0 && proj.y <= 1.0 && + proj.z >= 0.0 && proj.z <= 1.0) { + float bias = max(0.0005 * (1.0 - dot(norm, ldir)), 0.00005); shadow = texture(uShadowMap, vec3(proj.xy, proj.z - bias)); } shadow = mix(1.0, shadow, shadowParams.y); diff --git a/include/rendering/renderer.hpp b/include/rendering/renderer.hpp index 2df22e05..7447ecb8 100644 --- a/include/rendering/renderer.hpp +++ b/include/rendering/renderer.hpp @@ -233,6 +233,7 @@ private: VkSampler shadowSampler = VK_NULL_HANDLE; VkRenderPass shadowRenderPass = VK_NULL_HANDLE; VkFramebuffer shadowFramebuffer = VK_NULL_HANDLE; + VkImageLayout shadowDepthLayout_ = VK_IMAGE_LAYOUT_UNDEFINED; glm::mat4 lightSpaceMatrix = glm::mat4(1.0f); glm::vec3 shadowCenter = glm::vec3(0.0f); bool shadowCenterInitialized = false; diff --git a/src/rendering/character_renderer.cpp b/src/rendering/character_renderer.cpp index 56265fca..b6198113 100644 --- a/src/rendering/character_renderer.cpp +++ b/src/rendering/character_renderer.cpp @@ -2087,9 +2087,9 @@ bool CharacterRenderer::initializeShadow(VkRenderPass shadowRenderPass) { fragShader.stageInfo(VK_SHADER_STAGE_FRAGMENT_BIT)) .setVertexInput({vertBind}, vertAttrs) .setTopology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST) - .setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_FRONT_BIT) + .setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_NONE) .setDepthTest(true, true, VK_COMPARE_OP_LESS_OR_EQUAL) - .setDepthBias(2.0f, 4.0f) + .setDepthBias(0.05f, 0.20f) .setNoColorAttachment() .setLayout(shadowPipelineLayout_) .setRenderPass(shadowRenderPass) diff --git a/src/rendering/m2_renderer.cpp b/src/rendering/m2_renderer.cpp index dd4f3ac0..12e11ed5 100644 --- a/src/rendering/m2_renderer.cpp +++ b/src/rendering/m2_renderer.cpp @@ -2603,9 +2603,11 @@ bool M2Renderer::initializeShadow(VkRenderPass shadowRenderPass) { fragShader.stageInfo(VK_SHADER_STAGE_FRAGMENT_BIT)) .setVertexInput({vertBind}, vertAttrs) .setTopology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST) - .setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_FRONT_BIT) + // Foliage/leaf cards are effectively two-sided; front-face culling can + // drop them from the shadow map depending on light/view orientation. + .setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_NONE) .setDepthTest(true, true, VK_COMPARE_OP_LESS_OR_EQUAL) - .setDepthBias(2.0f, 4.0f) + .setDepthBias(0.05f, 0.20f) .setNoColorAttachment() .setLayout(shadowPipelineLayout_) .setRenderPass(shadowRenderPass) @@ -2655,9 +2657,10 @@ void M2Renderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceMa vkCmdPushConstants(cmd, shadowPipelineLayout_, VK_SHADER_STAGE_VERTEX_BIT, 0, 128, &push); - // Draw only opaque batches + // Draw all batches in shadow pass. + // Blend-mode filtering was excluding many valid world casters after + // Vulkan material path changes (trees/buildings losing shadows). for (const auto& batch : model.batches) { - if (batch.blendMode >= 2) continue; // skip transparent if (batch.submeshLevel > 0) continue; // skip LOD submeshes vkCmdDrawIndexed(cmd, batch.indexCount, 1, batch.indexStart, 0, 0); } diff --git a/src/rendering/renderer.cpp b/src/rendering/renderer.cpp index 1e714d21..80b6fd87 100644 --- a/src/rendering/renderer.cpp +++ b/src/rendering/renderer.cpp @@ -288,6 +288,7 @@ bool Renderer::createPerFrameResources() { LOG_ERROR("Failed to create shadow depth image"); return false; } + shadowDepthLayout_ = VK_IMAGE_LAYOUT_UNDEFINED; // --- Create shadow depth image view --- VkImageViewCreateInfo viewCI{}; @@ -304,15 +305,15 @@ bool Renderer::createPerFrameResources() { // --- Create shadow sampler --- VkSamplerCreateInfo sampCI{}; sampCI.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO; - sampCI.magFilter = VK_FILTER_LINEAR; - sampCI.minFilter = VK_FILTER_LINEAR; + sampCI.magFilter = VK_FILTER_NEAREST; + sampCI.minFilter = VK_FILTER_NEAREST; sampCI.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; sampCI.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; sampCI.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; sampCI.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; sampCI.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE; sampCI.compareEnable = VK_TRUE; - sampCI.compareOp = VK_COMPARE_OP_LESS_OR_EQUAL; + sampCI.compareOp = VK_COMPARE_OP_GREATER_OR_EQUAL; if (vkCreateSampler(device, &sampCI, nullptr, &shadowSampler) != VK_SUCCESS) { LOG_ERROR("Failed to create shadow sampler"); return false; @@ -326,7 +327,7 @@ bool Renderer::createPerFrameResources() { depthAtt.storeOp = VK_ATTACHMENT_STORE_OP_STORE; depthAtt.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; depthAtt.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; - depthAtt.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + depthAtt.initialLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; depthAtt.finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; VkAttachmentReference depthRef{}; @@ -501,6 +502,7 @@ void Renderer::destroyPerFrameResources() { if (shadowDepthView) { vkDestroyImageView(device, shadowDepthView, nullptr); shadowDepthView = VK_NULL_HANDLE; } if (shadowDepthImage) { vmaDestroyImage(vkCtx->getAllocator(), shadowDepthImage, shadowDepthAlloc); shadowDepthImage = VK_NULL_HANDLE; shadowDepthAlloc = VK_NULL_HANDLE; } if (shadowSampler) { vkDestroySampler(device, shadowSampler, nullptr); shadowSampler = VK_NULL_HANDLE; } + shadowDepthLayout_ = VK_IMAGE_LAYOUT_UNDEFINED; } void Renderer::updatePerFrameUBO() { @@ -3699,19 +3701,25 @@ void Renderer::renderShadowPass() { ubo->shadowParams = glm::vec4(shadowsEnabled ? 1.0f : 0.0f, 0.8f, 0.0f, 0.0f); } - // Barrier 1: UNDEFINED → DEPTH_STENCIL_ATTACHMENT_OPTIMAL + // Barrier 1: transition shadow map into writable depth layout. VkImageMemoryBarrier b1{}; b1.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; - b1.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + b1.oldLayout = shadowDepthLayout_; b1.newLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; b1.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; b1.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - b1.srcAccessMask = 0; - b1.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; + b1.srcAccessMask = (shadowDepthLayout_ == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) + ? VK_ACCESS_SHADER_READ_BIT + : 0; + b1.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; b1.image = shadowDepthImage; b1.subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, 1}; + VkPipelineStageFlags srcStage = (shadowDepthLayout_ == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) + ? VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT + : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; vkCmdPipelineBarrier(currentCmd, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT, + srcStage, VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT, 0, 0, nullptr, 0, nullptr, 1, &b1); // Begin shadow render pass @@ -3758,6 +3766,7 @@ void Renderer::renderShadowPass() { vkCmdPipelineBarrier(currentCmd, VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, 0, nullptr, 1, &b2); + shadowDepthLayout_ = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; } } // namespace rendering diff --git a/src/rendering/vk_texture.cpp b/src/rendering/vk_texture.cpp index 5c5cc5cc..fba6d72b 100644 --- a/src/rendering/vk_texture.cpp +++ b/src/rendering/vk_texture.cpp @@ -249,14 +249,14 @@ bool VkTexture::createSampler(VkDevice device, bool VkTexture::createShadowSampler(VkDevice device) { VkSamplerCreateInfo samplerInfo{}; samplerInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO; - samplerInfo.minFilter = VK_FILTER_LINEAR; - samplerInfo.magFilter = VK_FILTER_LINEAR; + samplerInfo.minFilter = VK_FILTER_NEAREST; + samplerInfo.magFilter = VK_FILTER_NEAREST; samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; samplerInfo.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE; samplerInfo.compareEnable = VK_TRUE; - samplerInfo.compareOp = VK_COMPARE_OP_LESS_OR_EQUAL; + samplerInfo.compareOp = VK_COMPARE_OP_GREATER_OR_EQUAL; samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; samplerInfo.minLod = 0.0f; samplerInfo.maxLod = 1.0f; diff --git a/src/rendering/wmo_renderer.cpp b/src/rendering/wmo_renderer.cpp index 15705a05..b770ac08 100644 --- a/src/rendering/wmo_renderer.cpp +++ b/src/rendering/wmo_renderer.cpp @@ -1486,9 +1486,9 @@ bool WMORenderer::initializeShadow(VkRenderPass shadowRenderPass) { fragShader.stageInfo(VK_SHADER_STAGE_FRAGMENT_BIT)) .setVertexInput({vertBind}, vertAttrs) .setTopology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST) - .setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_FRONT_BIT) + .setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_NONE) .setDepthTest(true, true, VK_COMPARE_OP_LESS_OR_EQUAL) - .setDepthBias(2.0f, 4.0f) + .setDepthBias(0.05f, 0.20f) .setNoColorAttachment() .setLayout(shadowPipelineLayout_) .setRenderPass(shadowRenderPass) @@ -1535,9 +1535,10 @@ void WMORenderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceM vkCmdBindVertexBuffers(cmd, 0, 1, &group.vertexBuffer, &offset); vkCmdBindIndexBuffer(cmd, group.indexBuffer, 0, VK_INDEX_TYPE_UINT16); - // Draw only opaque batches (skip transparent) + // Draw all batches in shadow pass. + // WMO transparency classification is not reliable enough for caster + // selection here and was dropping major world casters. for (const auto& mb : group.mergedBatches) { - if (mb.isTransparent) continue; for (const auto& dr : mb.draws) { vkCmdDrawIndexed(cmd, dr.indexCount, 1, dr.firstIndex, 0, 0); }