From 1fab17e639cc544d9f2c642740fba3e3087c7c2b Mon Sep 17 00:00:00 2001 From: Kelsidavis Date: Wed, 25 Feb 2026 08:18:26 -0800 Subject: [PATCH] Add Windows build scripts, fix multi-threaded MPQ extraction, and cross-platform temp paths - Add build.ps1/bat, rebuild.ps1/bat, debug_texture.ps1/bat (Windows equivalents of existing bash scripts, using directory junctions for Data link) - Fix asset extractor: StormLib is not thread-safe even with separate handles per thread. Serialize all MPQ reads behind a mutex while keeping CRC computation and disk writes parallel. Previously caused 99.8% extraction failures with >1 thread. - Add SFileHasFile() check during enumeration to skip listfile-only entries - Add diagnostic logging for extraction failures (first 5 per thread + summary) - Use std::filesystem::temp_directory_path() instead of hardcoded /tmp/ in character_renderer.cpp debug dumps - Update debug_texture.sh to use $TMPDIR fallback and glob for actual dump filenames --- build.bat | 3 + build.ps1 | 44 ++++++++ debug_texture.bat | 3 + debug_texture.ps1 | 64 +++++++++++ debug_texture.sh | 43 +++++--- rebuild.bat | 3 + rebuild.ps1 | 50 +++++++++ src/rendering/character_renderer.cpp | 6 +- tools/asset_extract/extractor.cpp | 152 ++++++++++++++++++--------- 9 files changed, 300 insertions(+), 68 deletions(-) create mode 100644 build.bat create mode 100644 build.ps1 create mode 100644 debug_texture.bat create mode 100644 debug_texture.ps1 create mode 100644 rebuild.bat create mode 100644 rebuild.ps1 diff --git a/build.bat b/build.bat new file mode 100644 index 00000000..efeefb5a --- /dev/null +++ b/build.bat @@ -0,0 +1,3 @@ +@echo off +REM Convenience wrapper — launches the PowerShell build script. +powershell -ExecutionPolicy Bypass -File "%~dp0build.ps1" %* diff --git a/build.ps1 b/build.ps1 new file mode 100644 index 00000000..1f35f0f1 --- /dev/null +++ b/build.ps1 @@ -0,0 +1,44 @@ +<# +.SYNOPSIS + Builds the wowee project (Windows equivalent of build.sh). + +.DESCRIPTION + Creates a build directory, runs CMake configure + build, and creates a + directory junction for the Data folder so the binary can find assets. +#> + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +Set-Location $ScriptDir + +Write-Host "Building wowee..." + +# Create build directory if it doesn't exist +if (-not (Test-Path "build")) { + New-Item -ItemType Directory -Path "build" | Out-Null +} +Set-Location "build" + +# Configure with CMake +Write-Host "Configuring with CMake..." +& cmake .. -DCMAKE_BUILD_TYPE=Release +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +# Build with all cores +$numProcs = $env:NUMBER_OF_PROCESSORS +if (-not $numProcs) { $numProcs = 4 } +Write-Host "Building with $numProcs cores..." +& cmake --build . --parallel $numProcs +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +# Ensure Data junction exists in bin directory +$binData = Join-Path (Get-Location) "bin\Data" +if (-not (Test-Path $binData)) { + $target = (Resolve-Path (Join-Path (Get-Location) "..\Data")).Path + cmd /c mklink /J "$binData" "$target" +} + +Write-Host "" +Write-Host "Build complete! Binary: build\bin\wowee.exe" +Write-Host "Run with: cd build\bin && .\wowee.exe" diff --git a/debug_texture.bat b/debug_texture.bat new file mode 100644 index 00000000..7e80310c --- /dev/null +++ b/debug_texture.bat @@ -0,0 +1,3 @@ +@echo off +REM Convenience wrapper — launches the PowerShell texture debug script. +powershell -ExecutionPolicy Bypass -File "%~dp0debug_texture.ps1" %* diff --git a/debug_texture.ps1 b/debug_texture.ps1 new file mode 100644 index 00000000..18feda3c --- /dev/null +++ b/debug_texture.ps1 @@ -0,0 +1,64 @@ +<# +.SYNOPSIS + Converts raw RGBA texture dumps to PNG for visual inspection (Windows equivalent of debug_texture.sh). + +.PARAMETER Width + Texture width in pixels. Defaults to 1024. + +.PARAMETER Height + Texture height in pixels. Defaults to 1024. + +.EXAMPLE + .\debug_texture.ps1 + .\debug_texture.ps1 -Width 2048 -Height 2048 +#> + +param( + [int]$Width = 1024, + [int]$Height = 1024 +) + +$TempDir = $env:TEMP + +Write-Host "Converting debug textures (${Width}x${Height})..." + +# Find raw dumps — filenames include dimensions (e.g. wowee_composite_debug_1024x1024.raw) +$rawFiles = Get-ChildItem -Path $TempDir -Filter "wowee_*_debug*.raw" -ErrorAction SilentlyContinue + +if (-not $rawFiles) { + Write-Host "No debug dumps found in $TempDir" + Write-Host " (looked for $TempDir\wowee_*_debug*.raw)" + exit 0 +} + +foreach ($rawItem in $rawFiles) { + $raw = $rawItem.FullName + $png = $raw -replace '\.raw$', '.png' + + # Try ImageMagick first, fall back to ffmpeg + if (Get-Command magick -ErrorAction SilentlyContinue) { + & magick -size "${Width}x${Height}" -depth 8 "rgba:$raw" "$png" 2>$null + if ($LASTEXITCODE -eq 0) { + Write-Host "Created $png (${Width}x${Height})" + } else { + Write-Host "Failed to convert $raw" + } + } elseif (Get-Command convert -ErrorAction SilentlyContinue) { + & convert -size "${Width}x${Height}" -depth 8 "rgba:$raw" "$png" 2>$null + if ($LASTEXITCODE -eq 0) { + Write-Host "Created $png (${Width}x${Height})" + } else { + Write-Host "Failed to convert $raw" + } + } elseif (Get-Command ffmpeg -ErrorAction SilentlyContinue) { + & ffmpeg -y -f rawvideo -pix_fmt rgba -s "${Width}x${Height}" -i "$raw" "$png" 2>$null + if ($LASTEXITCODE -eq 0) { + Write-Host "Created $png (${Width}x${Height})" + } else { + Write-Host "Failed to convert $raw" + } + } else { + Write-Host "Need 'magick' (ImageMagick) or 'ffmpeg' to convert $raw" + Write-Host " Install: winget install ImageMagick.ImageMagick" + } +} diff --git a/debug_texture.sh b/debug_texture.sh index feaa42d5..36725cd6 100755 --- a/debug_texture.sh +++ b/debug_texture.sh @@ -8,23 +8,32 @@ H=${2:-1024} echo "Converting debug textures (${W}x${H})..." -for raw in /tmp/wowee_composite_debug.raw /tmp/wowee_equip_composite_debug.raw; do - if [ -f "$raw" ]; then - png="${raw%.raw}.png" - # Try ImageMagick first, fall back to ffmpeg - if command -v convert &>/dev/null; then - convert -size ${W}x${H} -depth 8 rgba:"$raw" "$png" 2>/dev/null && \ - echo "Created $png (${W}x${H})" || \ - echo "Failed to convert $raw" - elif command -v ffmpeg &>/dev/null; then - ffmpeg -y -f rawvideo -pix_fmt rgba -s ${W}x${H} -i "$raw" "$png" 2>/dev/null && \ - echo "Created $png (${W}x${H})" || \ - echo "Failed to convert $raw" - else - echo "Need 'convert' (ImageMagick) or 'ffmpeg' to convert $raw" - echo " Install: sudo apt install imagemagick" - fi +TMPD="${TMPDIR:-/tmp}" + +# Find raw dumps — filenames include dimensions (e.g. wowee_composite_debug_1024x1024.raw) +shopt -s nullglob +RAW_FILES=("$TMPD"/wowee_*_debug*.raw) +shopt -u nullglob + +if [ ${#RAW_FILES[@]} -eq 0 ]; then + echo "No debug dumps found in $TMPD" + echo " (looked for $TMPD/wowee_*_debug*.raw)" + exit 0 +fi + +for raw in "${RAW_FILES[@]}"; do + png="${raw%.raw}.png" + # Try ImageMagick first, fall back to ffmpeg + if command -v convert &>/dev/null; then + convert -size ${W}x${H} -depth 8 rgba:"$raw" "$png" 2>/dev/null && \ + echo "Created $png (${W}x${H})" || \ + echo "Failed to convert $raw" + elif command -v ffmpeg &>/dev/null; then + ffmpeg -y -f rawvideo -pix_fmt rgba -s ${W}x${H} -i "$raw" "$png" 2>/dev/null && \ + echo "Created $png (${W}x${H})" || \ + echo "Failed to convert $raw" else - echo "Not found: $raw" + echo "Need 'convert' (ImageMagick) or 'ffmpeg' to convert $raw" + echo " Install: sudo apt install imagemagick" fi done diff --git a/rebuild.bat b/rebuild.bat new file mode 100644 index 00000000..196b6eed --- /dev/null +++ b/rebuild.bat @@ -0,0 +1,3 @@ +@echo off +REM Convenience wrapper — launches the PowerShell clean rebuild script. +powershell -ExecutionPolicy Bypass -File "%~dp0rebuild.ps1" %* diff --git a/rebuild.ps1 b/rebuild.ps1 new file mode 100644 index 00000000..b365e3b4 --- /dev/null +++ b/rebuild.ps1 @@ -0,0 +1,50 @@ +<# +.SYNOPSIS + Clean rebuilds the wowee project (Windows equivalent of rebuild.sh). + +.DESCRIPTION + Removes the build directory, reconfigures from scratch, rebuilds, and + creates a directory junction for the Data folder. +#> + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +Set-Location $ScriptDir + +Write-Host "Clean rebuilding wowee..." + +# Remove build directory completely +if (Test-Path "build") { + Write-Host "Removing old build directory..." + Remove-Item -Recurse -Force "build" +} + +# Create fresh build directory +New-Item -ItemType Directory -Path "build" | Out-Null +Set-Location "build" + +# Configure with CMake +Write-Host "Configuring with CMake..." +& cmake .. -DCMAKE_BUILD_TYPE=Release +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +# Build with all cores +$numProcs = $env:NUMBER_OF_PROCESSORS +if (-not $numProcs) { $numProcs = 4 } +Write-Host "Building with $numProcs cores..." +& cmake --build . --parallel $numProcs +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +# Create Data junction in bin directory +Write-Host "Creating Data junction..." +$binData = Join-Path (Get-Location) "bin\Data" +if (-not (Test-Path $binData)) { + $target = (Resolve-Path (Join-Path (Get-Location) "..\Data")).Path + cmd /c mklink /J "$binData" "$target" + Write-Host " Created Data junction -> $target" +} + +Write-Host "" +Write-Host "Clean build complete! Binary: build\bin\wowee.exe" +Write-Host "Run with: cd build\bin && .\wowee.exe" diff --git a/src/rendering/character_renderer.cpp b/src/rendering/character_renderer.cpp index 33ff425a..df5bac46 100644 --- a/src/rendering/character_renderer.cpp +++ b/src/rendering/character_renderer.cpp @@ -812,10 +812,10 @@ VkTexture* CharacterRenderer::compositeTextures(const std::vector& } } - // Debug: dump composite to /tmp for visual inspection + // Debug: dump composite to temp dir for visual inspection { - std::string dumpPath = "/tmp/wowee_composite_debug_" + - std::to_string(width) + "x" + std::to_string(height) + ".raw"; + std::string dumpPath = (std::filesystem::temp_directory_path() / ("wowee_composite_debug_" + + std::to_string(width) + "x" + std::to_string(height) + ".raw")).string(); std::ofstream dump(dumpPath, std::ios::binary); if (dump) { dump.write(reinterpret_cast(composite.data()), diff --git a/tools/asset_extract/extractor.cpp b/tools/asset_extract/extractor.cpp index b4b6c9d5..615b43b9 100644 --- a/tools/asset_extract/extractor.cpp +++ b/tools/asset_extract/extractor.cpp @@ -548,6 +548,12 @@ bool Extractor::enumerateFiles(const Options& opts, continue; } + // Verify file actually exists in this archive's hash table + // (listfiles can reference files from other archives) + if (!SFileHasFile(hMpq, fileName.c_str())) { + continue; + } + std::string norm = normalizeWowPath(fileName); if (opts.onlyUsedDbcs && !wantedDbcs.empty() && !wantedDbcs.contains(norm)) { continue; @@ -624,24 +630,36 @@ bool Extractor::run(const Options& opts) { std::atomic fileIndex{0}; size_t totalFiles = files.size(); + // Open archives ONCE in main thread — StormLib has global state that is not + // thread-safe even with separate handles, so we serialize all MPQ reads. + struct SharedArchive { + HANDLE handle; + int priority; + std::string path; + }; + std::vector sharedHandles; + for (const auto& ad : archives) { + HANDLE h = nullptr; + if (SFileOpenArchive(ad.path.c_str(), 0, 0, &h)) { + sharedHandles.push_back({h, ad.priority, ad.path}); + } else { + std::cerr << " Failed to open archive: " << ad.path << "\n"; + } + } + if (sharedHandles.empty()) { + std::cerr << "Failed to open any archives for extraction\n"; + return false; + } + if (sharedHandles.size() < archives.size()) { + std::cerr << " Opened " << sharedHandles.size() + << "/" << archives.size() << " archives\n"; + } + + // Mutex protecting all StormLib calls (open/read/close are not thread-safe) + std::mutex mpqMutex; + auto workerFn = [&]() { - // Each thread opens ALL archives independently (StormLib is not thread-safe per handle). - // Sorted highest-priority last, so we iterate in reverse to find the winning version. - struct ThreadArchive { - HANDLE handle; - int priority; - }; - std::vector threadHandles; - for (const auto& ad : archives) { - HANDLE h = nullptr; - if (SFileOpenArchive(ad.path.c_str(), 0, 0, &h)) { - threadHandles.push_back({h, ad.priority}); - } - } - if (threadHandles.empty()) { - std::cerr << "Worker thread: failed to open any archives\n"; - return; - } + int failLogCount = 0; while (true) { size_t idx = fileIndex.fetch_add(1); @@ -654,35 +672,52 @@ bool Extractor::run(const Options& opts) { std::string mappedPath = PathMapper::mapPath(wowPath); std::string fullOutputPath = effectiveOutputDir + "/" + mappedPath; - // Search archives in reverse priority order (highest priority first) - HANDLE hFile = nullptr; - for (auto it = threadHandles.rbegin(); it != threadHandles.rend(); ++it) { - if (SFileOpenFileEx(it->handle, wowPath.c_str(), 0, &hFile)) { - break; + // Read file data from MPQ under lock + std::vector data; + { + std::lock_guard lock(mpqMutex); + + // Search archives in reverse priority order (highest priority first) + HANDLE hFile = nullptr; + for (auto it = sharedHandles.rbegin(); it != sharedHandles.rend(); ++it) { + if (SFileOpenFileEx(it->handle, wowPath.c_str(), 0, &hFile)) { + break; + } + hFile = nullptr; + } + if (!hFile) { + stats.filesFailed++; + if (failLogCount < 5) { + failLogCount++; + std::cerr << " FAILED open: " << wowPath + << " (tried " << sharedHandles.size() << " archives)\n"; + } + continue; } - hFile = nullptr; - } - if (!hFile) { - stats.filesFailed++; - continue; - } - DWORD fileSize = SFileGetFileSize(hFile, nullptr); - if (fileSize == SFILE_INVALID_SIZE || fileSize == 0) { - SFileCloseFile(hFile); - stats.filesSkipped++; - continue; - } + DWORD fileSize = SFileGetFileSize(hFile, nullptr); + if (fileSize == SFILE_INVALID_SIZE || fileSize == 0) { + SFileCloseFile(hFile); + stats.filesSkipped++; + continue; + } - std::vector data(fileSize); - DWORD bytesRead = 0; - if (!SFileReadFile(hFile, data.data(), fileSize, &bytesRead, nullptr)) { + data.resize(fileSize); + DWORD bytesRead = 0; + if (!SFileReadFile(hFile, data.data(), fileSize, &bytesRead, nullptr)) { + SFileCloseFile(hFile); + stats.filesFailed++; + if (failLogCount < 5) { + failLogCount++; + std::cerr << " FAILED read: " << wowPath + << " (size=" << fileSize << ")\n"; + } + continue; + } SFileCloseFile(hFile); - stats.filesFailed++; - continue; + data.resize(bytesRead); } - SFileCloseFile(hFile); - data.resize(bytesRead); + // Lock released — CRC computation and disk write happen in parallel // Compute CRC32 uint32_t crc = ManifestWriter::computeCRC32(data.data(), data.size()); @@ -694,6 +729,11 @@ bool Extractor::run(const Options& opts) { std::ofstream out(fullOutputPath, std::ios::binary); if (!out.is_open()) { stats.filesFailed++; + if (failLogCount < 5) { + failLogCount++; + std::lock_guard lock(manifestMutex); + std::cerr << " FAILED write: " << fullOutputPath << "\n"; + } continue; } out.write(reinterpret_cast(data.data()), data.size()); @@ -721,10 +761,6 @@ bool Extractor::run(const Options& opts) { << std::flush; } } - - for (auto& th : threadHandles) { - SFileCloseArchive(th.handle); - } }; std::cout << "Extracting " << totalFiles << " files using " << numThreads << " threads...\n"; @@ -737,10 +773,30 @@ bool Extractor::run(const Options& opts) { t.join(); } - std::cout << "\r Extracted " << stats.filesExtracted.load() << " files (" + // Close archives (opened once in main thread) + for (auto& sh : sharedHandles) { + SFileCloseArchive(sh.handle); + } + + auto extracted = stats.filesExtracted.load(); + auto failed = stats.filesFailed.load(); + auto skipped = stats.filesSkipped.load(); + std::cout << "\n Extracted " << extracted << " files (" << stats.bytesExtracted.load() / (1024 * 1024) << " MB), " - << stats.filesSkipped.load() << " skipped, " - << stats.filesFailed.load() << " failed\n"; + << skipped << " skipped, " + << failed << " failed\n"; + + // If most files failed, print a diagnostic hint + if (failed > 0 && failed > extracted * 10) { + std::cerr << "\nWARNING: " << failed << " out of " << totalFiles + << " files failed to extract.\n" + << " This usually means worker threads could not open one or more MPQ archives.\n" + << " Common causes:\n" + << " - MPQ files on a network/external drive with access restrictions\n" + << " - Another program (WoW client, antivirus) has the MPQ files locked\n" + << " - Too many threads for the OS file-handle limit (try --threads 1)\n" + << " Re-run with --verbose for detailed diagnostics.\n"; + } // Merge with existing manifest so partial extractions don't nuke prior entries std::string manifestPath = effectiveOutputDir + "/manifest.json";