phaneron/Kelsidavis-WoWee
1
0
Fork 0
mirror of https://github.com/Kelsidavis/WoWee.git synced 2026-03-22 23:30:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Stabilize Vulkan shadow pipeline diagnostics and compatibility path

Browse source 
- Fix shadow depth image layout transitions by tracking per-frame old/new layouts.
- Update receiver shadow projection to Vulkan clip-depth convention.
- Test inverted shadow compare op path (GREATER_OR_EQUAL).
- Switch shadow compare samplers to NEAREST filtering for broader Vulkan compatibility.
- Expand shadow caster coverage by disabling caster cull filtering in WMO/M2/Character shadow pipelines.
- Keep light-space matrix path on stable character-centered framing.
This commit is contained in:
Kelsi 2026-02-22 10:23:20 -08:00
parent 2c5e0dd313
commit 67e63653a4
10 changed files with 55 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

9
assets/shaders/character.frag.glsl
View file

@ -63,9 +63,12 @@ void main() {
float shadow = 1.0;
if (shadowParams.x > 0.5) {
vec4 lsPos = lightSpaceMatrix * vec4(FragPos, 1.0);
vec3 proj = lsPos.xyz / lsPos.w * 0.5 + 0.5;
if (proj.z <= 1.0) {
float bias = max(0.005 * (1.0 - dot(norm, ldir)), 0.001);
vec3 proj = lsPos.xyz / lsPos.w;
proj.xy = proj.xy * 0.5 + 0.5;
if (proj.x >= 0.0 && proj.x <= 1.0 &&
proj.y >= 0.0 && proj.y <= 1.0 &&
proj.z >= 0.0 && proj.z <= 1.0) {
float bias = max(0.0005 * (1.0 - dot(norm, ldir)), 0.00005);
shadow = texture(uShadowMap, vec3(proj.xy, proj.z - bias));
}
shadow = mix(1.0, shadow, shadowParams.y);

9
assets/shaders/m2.frag.glsl
View file

@ -86,9 +86,12 @@ void main() {
float shadow = 1.0;
if (shadowParams.x > 0.5) {
vec4 lsPos = lightSpaceMatrix * vec4(FragPos, 1.0);
vec3 proj = lsPos.xyz / lsPos.w * 0.5 + 0.5;
if (proj.z <= 1.0) {
float bias = max(0.005 * (1.0 - abs(dot(norm, ldir))), 0.001);
vec3 proj = lsPos.xyz / lsPos.w;
proj.xy = proj.xy * 0.5 + 0.5;
if (proj.x >= 0.0 && proj.x <= 1.0 &&
proj.y >= 0.0 && proj.y <= 1.0 &&
proj.z >= 0.0 && proj.z <= 1.0) {
float bias = max(0.0005 * (1.0 - abs(dot(norm, ldir))), 0.00005);
shadow = texture(uShadowMap, vec3(proj.xy, proj.z - bias));
}
shadow = mix(1.0, shadow, shadowParams.y);

2
assets/shaders/terrain.frag.glsl
View file

@ -84,7 +84,7 @@ void main() {
vec3 proj = lsPos.xyz / lsPos.w;
proj.xy = proj.xy * 0.5 + 0.5;
if (proj.x >= 0.0 && proj.x <= 1.0 && proj.y >= 0.0 && proj.y <= 1.0 && proj.z <= 1.0) {
float bias = 0.002;
float bias = 0.0002;
shadow = texture(uShadowMap, vec3(proj.xy, proj.z - bias));
shadow = mix(1.0, shadow, shadowParams.y);
}

9
assets/shaders/wmo.frag.glsl
View file

@ -57,9 +57,12 @@ void main() {
float shadow = 1.0;
if (shadowParams.x > 0.5) {
vec4 lsPos = lightSpaceMatrix * vec4(FragPos, 1.0);
vec3 proj = lsPos.xyz / lsPos.w * 0.5 + 0.5;
if (proj.z <= 1.0) {
float bias = max(0.005 * (1.0 - dot(norm, ldir)), 0.001);
vec3 proj = lsPos.xyz / lsPos.w;
proj.xy = proj.xy * 0.5 + 0.5;
if (proj.x >= 0.0 && proj.x <= 1.0 &&
proj.y >= 0.0 && proj.y <= 1.0 &&
proj.z >= 0.0 && proj.z <= 1.0) {
float bias = max(0.0005 * (1.0 - dot(norm, ldir)), 0.00005);
shadow = texture(uShadowMap, vec3(proj.xy, proj.z - bias));
}
shadow = mix(1.0, shadow, shadowParams.y);

1
include/rendering/renderer.hpp
View file

@ -233,6 +233,7 @@ private:
VkSampler shadowSampler = VK_NULL_HANDLE;
VkRenderPass shadowRenderPass = VK_NULL_HANDLE;
VkFramebuffer shadowFramebuffer = VK_NULL_HANDLE;
VkImageLayout shadowDepthLayout_ = VK_IMAGE_LAYOUT_UNDEFINED;
glm::mat4 lightSpaceMatrix = glm::mat4(1.0f);
glm::vec3 shadowCenter = glm::vec3(0.0f);
bool shadowCenterInitialized = false;

4
src/rendering/character_renderer.cpp
View file

@ -2087,9 +2087,9 @@ bool CharacterRenderer::initializeShadow(VkRenderPass shadowRenderPass) {
fragShader.stageInfo(VK_SHADER_STAGE_FRAGMENT_BIT))
.setVertexInput({vertBind}, vertAttrs)
.setTopology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST)
.setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_FRONT_BIT)
.setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_NONE)
.setDepthTest(true, true, VK_COMPARE_OP_LESS_OR_EQUAL)
.setDepthBias(2.0f, 4.0f)
.setDepthBias(0.05f, 0.20f)
.setNoColorAttachment()
.setLayout(shadowPipelineLayout_)
.setRenderPass(shadowRenderPass)

11
src/rendering/m2_renderer.cpp
View file

@ -2603,9 +2603,11 @@ bool M2Renderer::initializeShadow(VkRenderPass shadowRenderPass) {
fragShader.stageInfo(VK_SHADER_STAGE_FRAGMENT_BIT))
.setVertexInput({vertBind}, vertAttrs)
.setTopology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST)
.setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_FRONT_BIT)
// Foliage/leaf cards are effectively two-sided; front-face culling can
// drop them from the shadow map depending on light/view orientation.
.setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_NONE)
.setDepthTest(true, true, VK_COMPARE_OP_LESS_OR_EQUAL)
.setDepthBias(2.0f, 4.0f)
.setDepthBias(0.05f, 0.20f)
.setNoColorAttachment()
.setLayout(shadowPipelineLayout_)
.setRenderPass(shadowRenderPass)
@ -2655,9 +2657,10 @@ void M2Renderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceMa
vkCmdPushConstants(cmd, shadowPipelineLayout_, VK_SHADER_STAGE_VERTEX_BIT,
0, 128, &push);
// Draw only opaque batches
// Draw all batches in shadow pass.
// Blend-mode filtering was excluding many valid world casters after
// Vulkan material path changes (trees/buildings losing shadows).
for (const auto& batch : model.batches) {
if (batch.blendMode >= 2) continue; // skip transparent
if (batch.submeshLevel > 0) continue; // skip LOD submeshes
vkCmdDrawIndexed(cmd, batch.indexCount, 1, batch.indexStart, 0, 0);
}

27
src/rendering/renderer.cpp
View file

@ -288,6 +288,7 @@ bool Renderer::createPerFrameResources() {
LOG_ERROR("Failed to create shadow depth image");
return false;
}
shadowDepthLayout_ = VK_IMAGE_LAYOUT_UNDEFINED;
// --- Create shadow depth image view ---
VkImageViewCreateInfo viewCI{};
@ -304,15 +305,15 @@ bool Renderer::createPerFrameResources() {
// --- Create shadow sampler ---
VkSamplerCreateInfo sampCI{};
sampCI.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
sampCI.magFilter = VK_FILTER_LINEAR;
sampCI.minFilter = VK_FILTER_LINEAR;
sampCI.magFilter = VK_FILTER_NEAREST;
sampCI.minFilter = VK_FILTER_NEAREST;
sampCI.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
sampCI.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
sampCI.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
sampCI.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
sampCI.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
sampCI.compareEnable = VK_TRUE;
sampCI.compareOp = VK_COMPARE_OP_LESS_OR_EQUAL;
sampCI.compareOp = VK_COMPARE_OP_GREATER_OR_EQUAL;
if (vkCreateSampler(device, &sampCI, nullptr, &shadowSampler) != VK_SUCCESS) {
LOG_ERROR("Failed to create shadow sampler");
return false;
@ -326,7 +327,7 @@ bool Renderer::createPerFrameResources() {
depthAtt.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
depthAtt.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
depthAtt.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
depthAtt.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
depthAtt.initialLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
depthAtt.finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
VkAttachmentReference depthRef{};
@ -501,6 +502,7 @@ void Renderer::destroyPerFrameResources() {
if (shadowDepthView) { vkDestroyImageView(device, shadowDepthView, nullptr); shadowDepthView = VK_NULL_HANDLE; }
if (shadowDepthImage) { vmaDestroyImage(vkCtx->getAllocator(), shadowDepthImage, shadowDepthAlloc); shadowDepthImage = VK_NULL_HANDLE; shadowDepthAlloc = VK_NULL_HANDLE; }
if (shadowSampler) { vkDestroySampler(device, shadowSampler, nullptr); shadowSampler = VK_NULL_HANDLE; }
shadowDepthLayout_ = VK_IMAGE_LAYOUT_UNDEFINED;
}
void Renderer::updatePerFrameUBO() {
@ -3699,19 +3701,25 @@ void Renderer::renderShadowPass() {
ubo->shadowParams = glm::vec4(shadowsEnabled ? 1.0f : 0.0f, 0.8f, 0.0f, 0.0f);
}
// Barrier 1: UNDEFINED → DEPTH_STENCIL_ATTACHMENT_OPTIMAL
// Barrier 1: transition shadow map into writable depth layout.
VkImageMemoryBarrier b1{};
b1.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
b1.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
b1.oldLayout = shadowDepthLayout_;
b1.newLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
b1.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
b1.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
b1.srcAccessMask = 0;
b1.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
b1.srcAccessMask = (shadowDepthLayout_ == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
? VK_ACCESS_SHADER_READ_BIT
: 0;
b1.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT |
VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
b1.image = shadowDepthImage;
b1.subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, 1};
VkPipelineStageFlags srcStage = (shadowDepthLayout_ == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
? VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT
: VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
vkCmdPipelineBarrier(currentCmd,
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT,
srcStage, VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT,
0, 0, nullptr, 0, nullptr, 1, &b1);
// Begin shadow render pass
@ -3758,6 +3766,7 @@ void Renderer::renderShadowPass() {
vkCmdPipelineBarrier(currentCmd,
VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
0, 0, nullptr, 0, nullptr, 1, &b2);
shadowDepthLayout_ = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
}
} // namespace rendering

6
src/rendering/vk_texture.cpp
View file

@ -249,14 +249,14 @@ bool VkTexture::createSampler(VkDevice device,
bool VkTexture::createShadowSampler(VkDevice device) {
VkSamplerCreateInfo samplerInfo{};
samplerInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
samplerInfo.minFilter = VK_FILTER_LINEAR;
samplerInfo.magFilter = VK_FILTER_LINEAR;
samplerInfo.minFilter = VK_FILTER_NEAREST;
samplerInfo.magFilter = VK_FILTER_NEAREST;
samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
samplerInfo.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
samplerInfo.compareEnable = VK_TRUE;
samplerInfo.compareOp = VK_COMPARE_OP_LESS_OR_EQUAL;
samplerInfo.compareOp = VK_COMPARE_OP_GREATER_OR_EQUAL;
samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
samplerInfo.minLod = 0.0f;
samplerInfo.maxLod = 1.0f;

9
src/rendering/wmo_renderer.cpp
View file

@ -1486,9 +1486,9 @@ bool WMORenderer::initializeShadow(VkRenderPass shadowRenderPass) {
fragShader.stageInfo(VK_SHADER_STAGE_FRAGMENT_BIT))
.setVertexInput({vertBind}, vertAttrs)
.setTopology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST)
.setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_FRONT_BIT)
.setRasterization(VK_POLYGON_MODE_FILL, VK_CULL_MODE_NONE)
.setDepthTest(true, true, VK_COMPARE_OP_LESS_OR_EQUAL)
.setDepthBias(2.0f, 4.0f)
.setDepthBias(0.05f, 0.20f)
.setNoColorAttachment()
.setLayout(shadowPipelineLayout_)
.setRenderPass(shadowRenderPass)
@ -1535,9 +1535,10 @@ void WMORenderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceM
vkCmdBindVertexBuffers(cmd, 0, 1, &group.vertexBuffer, &offset);
vkCmdBindIndexBuffer(cmd, group.indexBuffer, 0, VK_INDEX_TYPE_UINT16);
// Draw only opaque batches (skip transparent)
// Draw all batches in shadow pass.
// WMO transparency classification is not reliable enough for caster
// selection here and was dropping major world casters.
for (const auto& mb : group.mergedBatches) {
if (mb.isTransparent) continue;
for (const auto& dr : mb.draws) {
vkCmdDrawIndexed(cmd, dr.indexCount, 1, dr.firstIndex, 0, 0);
}