From 4cae4bfcdc5a9d94fac17d701241b03491477af5 Mon Sep 17 00:00:00 2001
From: Kelsi <kelsihates2fa@gmail.com>
Date: Fri, 6 Mar 2026 19:21:48 -0800
Subject: [PATCH] Fix WMO shadow culling: use AABB instead of origin point
 distance

WMO origins can be far from their visible geometry, causing large city
buildings to be culled from the shadow pass. Use world bounding box for
instance culling and per-group AABB culling. Also increase WMO shadow
cull radius to match the shadow map coverage (180 units).
---
 src/rendering/wmo_renderer.cpp | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/src/rendering/wmo_renderer.cpp b/src/rendering/wmo_renderer.cpp
index ce4d4ab1..5c8be756 100644
--- a/src/rendering/wmo_renderer.cpp
+++ b/src/rendering/wmo_renderer.cpp
@@ -1724,11 +1724,18 @@ void WMORenderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceM
 
     struct ShadowPush { glm::mat4 lightSpaceMatrix; glm::mat4 model; };
 
-    const float shadowRadiusSq = shadowRadius * shadowRadius;
+    // WMO shadow cull uses the ortho half-extent (shadow map coverage) rather than
+    // the proximity radius so that distant buildings whose shadows reach the player
+    // are still rendered into the shadow map.
+    const float wmoCullRadius = std::max(shadowRadius, 180.0f);
+    const float wmoCullRadiusSq = wmoCullRadius * wmoCullRadius;
+
     for (const auto& instance : instances) {
-        // Distance cull against shadow frustum
-        glm::vec3 diff = instance.position - shadowCenter;
-        if (glm::dot(diff, diff) > shadowRadiusSq) continue;
+        // Distance cull using world bounding box — WMO origins can be far from
+        // their geometry, so point-based culling misses large buildings.
+        glm::vec3 closest = glm::clamp(shadowCenter, instance.worldBoundsMin, instance.worldBoundsMax);
+        glm::vec3 diff = closest - shadowCenter;
+        if (glm::dot(diff, diff) > wmoCullRadiusSq) continue;
         auto modelIt = loadedModels.find(instance.modelId);
         if (modelIt == loadedModels.end()) continue;
         const ModelData& model = modelIt->second;
@@ -1737,7 +1744,8 @@ void WMORenderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceM
         vkCmdPushConstants(cmd, shadowPipelineLayout_, VK_SHADER_STAGE_VERTEX_BIT,
                            0, 128, &push);
 
-        for (const auto& group : model.groups) {
+        for (size_t gi = 0; gi < model.groups.size(); ++gi) {
+            const auto& group = model.groups[gi];
             if (group.vertexBuffer == VK_NULL_HANDLE || group.indexBuffer == VK_NULL_HANDLE) continue;
 
             // Skip antiportal geometry
@@ -1746,13 +1754,18 @@ void WMORenderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceM
             // Skip LOD groups in shadow pass (they overlap real geometry)
             if (group.isLOD) continue;
 
+            // Per-group AABB cull against shadow frustum
+            if (gi < instance.worldGroupBounds.size()) {
+                const auto& [gMin, gMax] = instance.worldGroupBounds[gi];
+                glm::vec3 gClosest = glm::clamp(shadowCenter, gMin, gMax);
+                glm::vec3 gDiff = gClosest - shadowCenter;
+                if (glm::dot(gDiff, gDiff) > wmoCullRadiusSq) continue;
+            }
+
             VkDeviceSize offset = 0;
             vkCmdBindVertexBuffers(cmd, 0, 1, &group.vertexBuffer, &offset);
             vkCmdBindIndexBuffer(cmd, group.indexBuffer, 0, VK_INDEX_TYPE_UINT16);
 
-            // Draw all batches in shadow pass.
-            // WMO transparency classification is not reliable enough for caster
-            // selection here and was dropping major world casters.
             for (const auto& mb : group.mergedBatches) {
                 for (const auto& dr : mb.draws) {
                     vkCmdDrawIndexed(cmd, dr.indexCount, 1, dr.firstIndex, 0, 0);