Add shader-driven tree beautification: wind sway, SSS, color variation, AO

- Vertex wind animation: 3-layer displacement (trunk/branch/leaf) with
  quadratic height scaling so bases stay grounded
- Shadow pass: matching vertex displacement split into foliage/non-foliage
  passes, removed UV-wiggle approach
- Leaf subsurface scattering: warm backlit glow when looking toward sun
- Per-instance color variation: hue/brightness from position hash via flat
  varying to avoid interpolation flicker
- Canopy ambient occlusion: height-based darkening of tree interiors
- Detail normal perturbation: UV-only procedural normals to break flat cards
- Bayer 4x4 ordered dither replacing sin-hash noise for alpha edges
- Foliage skips shadow map sampling and specular to prevent flicker from
  swaying geometry sampling unstable shadow/highlight values
This commit is contained in:
Kelsi 2026-02-23 03:53:50 -08:00
parent 4511de8d38
commit ef1e5abe8e
11 changed files with 199 additions and 69 deletions

View file

@ -32,12 +32,28 @@ layout(set = 0, binding = 1) uniform sampler2DShadow uShadowMap;
layout(location = 0) in vec3 FragPos;
layout(location = 1) in vec3 Normal;
layout(location = 2) in vec2 TexCoord;
layout(location = 3) flat in vec3 InstanceOrigin;
layout(location = 4) in float ModelHeight;
layout(location = 0) out vec4 outColor;
// 4x4 Bayer dither matrix (normalized to 0..1)
float bayerDither4x4(ivec2 p) {
int idx = (p.x & 3) + (p.y & 3) * 4;
float m[16] = float[16](
0.0/16.0, 8.0/16.0, 2.0/16.0, 10.0/16.0,
12.0/16.0, 4.0/16.0, 14.0/16.0, 6.0/16.0,
3.0/16.0, 11.0/16.0, 1.0/16.0, 9.0/16.0,
15.0/16.0, 7.0/16.0, 13.0/16.0, 5.0/16.0
);
return m[idx];
}
void main() {
vec4 texColor = hasTexture != 0 ? texture(uTexture, TexCoord) : vec4(1.0);
bool isFoliage = (alphaTest == 2);
float alphaCutoff = 0.5;
if (alphaTest == 2) {
// Vegetation cutout: lower threshold to preserve leaf coverage at grazing angles.
@ -50,13 +66,13 @@ void main() {
}
if (alphaTest == 2) {
float alpha = texColor.a;
float softBand = 0.12;
float softBand = 0.15;
if (alpha < (alphaCutoff - softBand)) discard;
if (alpha < alphaCutoff) {
vec2 p = floor(gl_FragCoord.xy);
float n = fract(sin(dot(p, vec2(12.9898, 78.233))) * 43758.5453);
ivec2 p = ivec2(gl_FragCoord.xy);
float threshold = bayerDither4x4(p);
float keep = clamp((alpha - (alphaCutoff - softBand)) / softBand, 0.0, 1.0);
if (n > keep) discard;
if (threshold > keep) discard;
}
} else if (alphaTest != 0 && texColor.a < alphaCutoff) {
discard;
@ -67,10 +83,26 @@ void main() {
}
if (blendMode == 1 && texColor.a < 0.004) discard;
// Per-instance color variation (foliage only)
if (isFoliage) {
float hash = fract(sin(dot(InstanceOrigin.xy, vec2(127.1, 311.7))) * 43758.5453);
float hueShiftR = 1.0 + (hash - 0.5) * 0.16; // ±8% red
float hueShiftB = 1.0 + (fract(hash * 7.13) - 0.5) * 0.16; // ±8% blue
float brightness = 0.85 + hash * 0.30; // 85115%
texColor.rgb *= vec3(hueShiftR, 1.0, hueShiftB) * brightness;
}
vec3 norm = normalize(Normal);
bool foliageTwoSided = (alphaTest == 2);
if (!foliageTwoSided && !gl_FrontFacing) norm = -norm;
// Detail normal perturbation (foliage only) — UV-based only so wind doesn't cause flicker
if (isFoliage) {
float nx = sin(TexCoord.x * 12.0 + TexCoord.y * 5.3) * 0.10;
float ny = sin(TexCoord.y * 14.0 + TexCoord.x * 4.7) * 0.10;
norm = normalize(norm + vec3(nx, ny, 0.0));
}
vec3 ldir = normalize(-lightDir.xyz);
float nDotL = dot(norm, ldir);
float diff = foliageTwoSided ? abs(nDotL) : max(nDotL, 0.0);
@ -80,10 +112,19 @@ void main() {
result = texColor.rgb;
} else {
vec3 viewDir = normalize(viewPos.xyz - FragPos);
vec3 halfDir = normalize(ldir + viewDir);
float spec = pow(max(dot(norm, halfDir), 0.0), 32.0) * specularIntensity;
// Foliage: no specular, no shadow map — both flicker on swaying thin cards
float spec = 0.0;
float shadow = 1.0;
if (isFoliage) {
// Use a fixed gentle shadow from the shadow system strength
if (shadowParams.x > 0.5) {
shadow = mix(1.0, 0.75, shadowParams.y);
}
} else {
vec3 halfDir = normalize(ldir + viewDir);
spec = pow(max(dot(norm, halfDir), 0.0), 32.0) * specularIntensity;
if (shadowParams.x > 0.5) {
vec4 lsPos = lightSpaceMatrix * vec4(FragPos, 1.0);
vec3 proj = lsPos.xyz / lsPos.w;
@ -95,17 +136,34 @@ void main() {
shadow = texture(uShadowMap, vec3(proj.xy, proj.z - bias));
}
shadow = mix(1.0, shadow, shadowParams.y);
if (foliageTwoSided) shadow = max(shadow, 0.45);
}
}
// Leaf subsurface scattering (foliage only) — uses stable normal, no FragPos dependency
vec3 sss = vec3(0.0);
if (isFoliage) {
float backLit = max(-nDotL, 0.0);
float viewDotLight = max(dot(viewDir, -ldir), 0.0);
float sssAmount = backLit * pow(viewDotLight, 4.0) * 0.35 * texColor.a;
sss = sssAmount * vec3(1.0, 0.9, 0.5) * lightColor.rgb;
}
result = ambientColor.rgb * texColor.rgb
+ shadow * (diff * lightColor.rgb * texColor.rgb + spec * lightColor.rgb);
+ shadow * (diff * lightColor.rgb * texColor.rgb + spec * lightColor.rgb)
+ sss;
if (interiorDarken > 0.0) {
result *= mix(1.0, 0.5, interiorDarken);
}
}
// Canopy ambient occlusion (foliage only)
if (isFoliage) {
float normalizedHeight = clamp(ModelHeight / 18.0, 0.0, 1.0);
float aoFactor = mix(0.55, 1.0, smoothstep(0.0, 0.6, normalizedHeight));
result *= aoFactor;
}
float dist = length(viewPos.xyz - FragPos);
float fogFactor = clamp((fogParams.y - dist) / (fogParams.y - fogParams.x), 0.0, 1.0);
result = mix(fogColor.rgb, result, fogFactor);

Binary file not shown.

View file

@ -18,6 +18,7 @@ layout(push_constant) uniform Push {
vec2 uvOffset;
int texCoordSet;
int useBones;
int isFoliage;
} push;
layout(set = 2, binding = 0) readonly buffer BoneSSBO {
@ -34,6 +35,8 @@ layout(location = 5) in vec2 aTexCoord2;
layout(location = 0) out vec3 FragPos;
layout(location = 1) out vec3 Normal;
layout(location = 2) out vec2 TexCoord;
layout(location = 3) flat out vec3 InstanceOrigin;
layout(location = 4) out float ModelHeight;
void main() {
vec4 pos = vec4(aPos, 1.0);
@ -49,11 +52,40 @@ void main() {
norm = skinMat * norm;
}
// Wind animation for foliage
if (push.isFoliage != 0) {
float windTime = fogParams.z;
vec3 worldRef = push.model[3].xyz;
float heightFactor = clamp(pos.z / 20.0, 0.0, 1.0);
heightFactor *= heightFactor; // quadratic — base stays grounded
// Layer 1: Trunk sway — slow, large amplitude
float trunkPhase = windTime * 0.8 + dot(worldRef.xy, vec2(0.1, 0.13));
float trunkSwayX = sin(trunkPhase) * 0.35 * heightFactor;
float trunkSwayY = cos(trunkPhase * 0.7) * 0.25 * heightFactor;
// Layer 2: Branch sway — medium frequency, per-branch phase
float branchPhase = windTime * 1.7 + dot(worldRef.xy, vec2(0.37, 0.71));
float branchSwayX = sin(branchPhase + pos.y * 0.4) * 0.15 * heightFactor;
float branchSwayY = cos(branchPhase * 1.1 + pos.x * 0.3) * 0.12 * heightFactor;
// Layer 3: Leaf flutter — fast, small amplitude, per-vertex
float leafPhase = windTime * 4.5 + dot(aPos, vec3(1.7, 2.3, 0.9));
float leafFlutterX = sin(leafPhase) * 0.06 * heightFactor;
float leafFlutterY = cos(leafPhase * 1.3) * 0.05 * heightFactor;
pos.x += trunkSwayX + branchSwayX + leafFlutterX;
pos.y += trunkSwayY + branchSwayY + leafFlutterY;
}
vec4 worldPos = push.model * pos;
FragPos = worldPos.xyz;
Normal = mat3(push.model) * norm.xyz;
TexCoord = (push.texCoordSet == 1 ? aTexCoord2 : aTexCoord) + push.uvOffset;
InstanceOrigin = push.model[3].xyz;
ModelHeight = pos.z;
gl_Position = projection * view * worldPos;
}

Binary file not shown.

View file

@ -16,22 +16,7 @@ layout(location = 1) in vec3 WorldPos;
void main() {
if (useTexture != 0) {
vec2 uv = TexCoord;
if (foliageSway != 0) {
float sway = sin(windTime + WorldPos.x * 0.5) * 0.02 * foliageMotionDamp;
uv += vec2(sway, sway * 0.5);
}
vec4 texColor = textureLod(uTexture, uv, 0.0);
vec4 texColor = textureLod(uTexture, TexCoord, 0.0);
if (alphaTest != 0 && texColor.a < 0.5) discard;
if (foliageSway != 0) {
vec2 uv2 = TexCoord + vec2(
sin(windTime * 1.3 + WorldPos.z * 0.7) * 0.015 * foliageMotionDamp,
sin(windTime * 0.9 + WorldPos.x * 0.6) * 0.01 * foliageMotionDamp
);
vec4 texColor2 = textureLod(uTexture, uv2, 0.0);
float blended = (texColor.a + texColor2.a) * 0.5;
if (alphaTest != 0 && blended < 0.5) discard;
}
}
}

Binary file not shown.

View file

@ -23,7 +23,34 @@ layout(location = 0) out vec2 TexCoord;
layout(location = 1) out vec3 WorldPos;
void main() {
vec4 worldPos = push.model * vec4(aPos, 1.0);
vec4 pos = vec4(aPos, 1.0);
// Wind vertex displacement for foliage (matches m2.vert.glsl)
if (foliageSway != 0) {
vec3 worldRef = push.model[3].xyz;
float heightFactor = clamp(pos.z / 20.0, 0.0, 1.0);
heightFactor *= heightFactor;
// Layer 1: Trunk sway
float trunkPhase = windTime * 0.8 + dot(worldRef.xy, vec2(0.1, 0.13));
float trunkSwayX = sin(trunkPhase) * 0.35 * heightFactor;
float trunkSwayY = cos(trunkPhase * 0.7) * 0.25 * heightFactor;
// Layer 2: Branch sway
float branchPhase = windTime * 1.7 + dot(worldRef.xy, vec2(0.37, 0.71));
float branchSwayX = sin(branchPhase + pos.y * 0.4) * 0.15 * heightFactor;
float branchSwayY = cos(branchPhase * 1.1 + pos.x * 0.3) * 0.12 * heightFactor;
// Layer 3: Leaf flutter
float leafPhase = windTime * 4.5 + dot(aPos, vec3(1.7, 2.3, 0.9));
float leafFlutterX = sin(leafPhase) * 0.06 * heightFactor;
float leafFlutterY = cos(leafPhase * 1.3) * 0.05 * heightFactor;
pos.x += trunkSwayX + branchSwayX + leafFlutterX;
pos.y += trunkSwayY + branchSwayY + leafFlutterY;
}
vec4 worldPos = push.model * pos;
WorldPos = worldPos.xyz;
TexCoord = aTexCoord;
gl_Position = push.lightSpaceMatrix * worldPos;

Binary file not shown.

View file

@ -251,7 +251,7 @@ public:
/**
* Render depth-only pass for shadow casting
*/
void renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceMatrix);
void renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceMatrix, float globalTime = 0.0f);
/**
* Render M2 particle emitters (point sprites)

View file

@ -401,7 +401,7 @@ bool M2Renderer::initialize(VkContext* ctx, VkDescriptorSetLayout perFrameLayout
VkPushConstantRange pushRange{};
pushRange.stageFlags = VK_SHADER_STAGE_VERTEX_BIT;
pushRange.offset = 0;
pushRange.size = 80; // mat4(64) + vec2(8) + int(4) + int(4)
pushRange.size = 84; // mat4(64) + vec2(8) + int(4) + int(4) + int(4)
VkPipelineLayoutCreateInfo ci{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
ci.setLayoutCount = 3;
@ -2109,6 +2109,7 @@ void M2Renderer::render(VkCommandBuffer cmd, VkDescriptorSet perFrameSet, const
glm::vec2 uvOffset;
int texCoordSet;
int useBones;
int isFoliage;
};
// Bind per-frame descriptor set (set 0) — shared across all draws
@ -2382,6 +2383,7 @@ void M2Renderer::render(VkCommandBuffer cmd, VkDescriptorSet perFrameSet, const
pc.uvOffset = uvOffset;
pc.texCoordSet = static_cast<int>(batch.textureUnit);
pc.useBones = useBones ? 1 : 0;
pc.isFoliage = model.shadowWindFoliage ? 1 : 0;
vkCmdPushConstants(cmd, pipelineLayout_, VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(pc), &pc);
vkCmdDrawIndexed(cmd, batch.indexCount, 1, batch.indexStart, 0, 0);
@ -2625,16 +2627,36 @@ bool M2Renderer::initializeShadow(VkRenderPass shadowRenderPass) {
return true;
}
void M2Renderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceMatrix) {
void M2Renderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceMatrix, float globalTime) {
if (!shadowPipeline_ || !shadowParamsSet_) return;
if (instances.empty() || models.empty()) return;
struct ShadowPush { glm::mat4 lightSpaceMatrix; glm::mat4 model; };
// Helper lambda to draw instances with a given foliageSway setting
auto drawPass = [&](bool foliagePass) {
// Update ShadowParams UBO for this pass
struct ShadowParamsUBO {
int32_t useBones = 0;
int32_t useTexture = 0;
int32_t alphaTest = 0;
int32_t foliageSway = 0;
float windTime = 0.0f;
float foliageMotionDamp = 1.0f;
};
ShadowParamsUBO params{};
params.foliageSway = foliagePass ? 1 : 0;
params.windTime = globalTime;
params.foliageMotionDamp = 1.0f;
VmaAllocationInfo allocInfo{};
vmaGetAllocationInfo(vkCtx_->getAllocator(), shadowParamsAlloc_, &allocInfo);
std::memcpy(allocInfo.pMappedData, &params, sizeof(params));
vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, shadowPipeline_);
vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, shadowPipelineLayout_,
0, 1, &shadowParamsSet_, 0, nullptr);
struct ShadowPush { glm::mat4 lightSpaceMatrix; glm::mat4 model; };
uint32_t currentModelId = UINT32_MAX;
const M2ModelGPU* currentModel = nullptr;
@ -2644,6 +2666,9 @@ void M2Renderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceMa
const M2ModelGPU& model = modelIt->second;
if (!model.isValid() || model.isSmoke || model.isInvisibleTrap) continue;
// Filter: only draw foliage models in foliage pass, non-foliage in non-foliage pass
if (model.shadowWindFoliage != foliagePass) continue;
// Bind vertex/index buffers when model changes
if (instance.modelId != currentModelId) {
currentModelId = instance.modelId;
@ -2657,14 +2682,17 @@ void M2Renderer::renderShadow(VkCommandBuffer cmd, const glm::mat4& lightSpaceMa
vkCmdPushConstants(cmd, shadowPipelineLayout_, VK_SHADER_STAGE_VERTEX_BIT,
0, 128, &push);
// Draw all batches in shadow pass.
// Blend-mode filtering was excluding many valid world casters after
// Vulkan material path changes (trees/buildings losing shadows).
for (const auto& batch : model.batches) {
if (batch.submeshLevel > 0) continue; // skip LOD submeshes
if (batch.submeshLevel > 0) continue;
vkCmdDrawIndexed(cmd, batch.indexCount, 1, batch.indexStart, 0, 0);
}
}
};
// Pass 1: non-foliage (no wind displacement)
drawPass(false);
// Pass 2: foliage (wind displacement enabled)
drawPass(true);
}
// --- M2 Particle Emitter Helpers ---

View file

@ -3975,7 +3975,7 @@ void Renderer::renderShadowPass() {
wmoRenderer->renderShadow(currentCmd, lightSpaceMatrix);
}
if (m2Renderer) {
m2Renderer->renderShadow(currentCmd, lightSpaceMatrix);
m2Renderer->renderShadow(currentCmd, lightSpaceMatrix, globalTime);
}
if (characterRenderer) {
characterRenderer->renderShadow(currentCmd, lightSpaceMatrix);