Add Windows build scripts, fix multi-threaded MPQ extraction, and cross-platform temp paths

- Add build.ps1/bat, rebuild.ps1/bat, debug_texture.ps1/bat (Windows equivalents
  of existing bash scripts, using directory junctions for Data link)
- Fix asset extractor: StormLib is not thread-safe even with separate handles per
  thread. Serialize all MPQ reads behind a mutex while keeping CRC computation and
  disk writes parallel. Previously caused 99.8% extraction failures with >1 thread.
- Add SFileHasFile() check during enumeration to skip listfile-only entries
- Add diagnostic logging for extraction failures (first 5 per thread + summary)
- Use std::filesystem::temp_directory_path() instead of hardcoded /tmp/ in
  character_renderer.cpp debug dumps
- Update debug_texture.sh to use $TMPDIR fallback and glob for actual dump filenames
This commit is contained in:
Kelsidavis 2026-02-25 08:18:26 -08:00
parent 570dec8b88
commit 1fab17e639
9 changed files with 300 additions and 68 deletions

3
build.bat Normal file
View file

@ -0,0 +1,3 @@
@echo off
REM Convenience wrapper — launches the PowerShell build script.
powershell -ExecutionPolicy Bypass -File "%~dp0build.ps1" %*

44
build.ps1 Normal file
View file

@ -0,0 +1,44 @@
<#
.SYNOPSIS
Builds the wowee project (Windows equivalent of build.sh).
.DESCRIPTION
Creates a build directory, runs CMake configure + build, and creates a
directory junction for the Data folder so the binary can find assets.
#>
$ErrorActionPreference = "Stop"
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
Set-Location $ScriptDir
Write-Host "Building wowee..."
# Create build directory if it doesn't exist
if (-not (Test-Path "build")) {
New-Item -ItemType Directory -Path "build" | Out-Null
}
Set-Location "build"
# Configure with CMake
Write-Host "Configuring with CMake..."
& cmake .. -DCMAKE_BUILD_TYPE=Release
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
# Build with all cores
$numProcs = $env:NUMBER_OF_PROCESSORS
if (-not $numProcs) { $numProcs = 4 }
Write-Host "Building with $numProcs cores..."
& cmake --build . --parallel $numProcs
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
# Ensure Data junction exists in bin directory
$binData = Join-Path (Get-Location) "bin\Data"
if (-not (Test-Path $binData)) {
$target = (Resolve-Path (Join-Path (Get-Location) "..\Data")).Path
cmd /c mklink /J "$binData" "$target"
}
Write-Host ""
Write-Host "Build complete! Binary: build\bin\wowee.exe"
Write-Host "Run with: cd build\bin && .\wowee.exe"

3
debug_texture.bat Normal file
View file

@ -0,0 +1,3 @@
@echo off
REM Convenience wrapper — launches the PowerShell texture debug script.
powershell -ExecutionPolicy Bypass -File "%~dp0debug_texture.ps1" %*

64
debug_texture.ps1 Normal file
View file

@ -0,0 +1,64 @@
<#
.SYNOPSIS
Converts raw RGBA texture dumps to PNG for visual inspection (Windows equivalent of debug_texture.sh).
.PARAMETER Width
Texture width in pixels. Defaults to 1024.
.PARAMETER Height
Texture height in pixels. Defaults to 1024.
.EXAMPLE
.\debug_texture.ps1
.\debug_texture.ps1 -Width 2048 -Height 2048
#>
param(
[int]$Width = 1024,
[int]$Height = 1024
)
$TempDir = $env:TEMP
Write-Host "Converting debug textures (${Width}x${Height})..."
# Find raw dumps — filenames include dimensions (e.g. wowee_composite_debug_1024x1024.raw)
$rawFiles = Get-ChildItem -Path $TempDir -Filter "wowee_*_debug*.raw" -ErrorAction SilentlyContinue
if (-not $rawFiles) {
Write-Host "No debug dumps found in $TempDir"
Write-Host " (looked for $TempDir\wowee_*_debug*.raw)"
exit 0
}
foreach ($rawItem in $rawFiles) {
$raw = $rawItem.FullName
$png = $raw -replace '\.raw$', '.png'
# Try ImageMagick first, fall back to ffmpeg
if (Get-Command magick -ErrorAction SilentlyContinue) {
& magick -size "${Width}x${Height}" -depth 8 "rgba:$raw" "$png" 2>$null
if ($LASTEXITCODE -eq 0) {
Write-Host "Created $png (${Width}x${Height})"
} else {
Write-Host "Failed to convert $raw"
}
} elseif (Get-Command convert -ErrorAction SilentlyContinue) {
& convert -size "${Width}x${Height}" -depth 8 "rgba:$raw" "$png" 2>$null
if ($LASTEXITCODE -eq 0) {
Write-Host "Created $png (${Width}x${Height})"
} else {
Write-Host "Failed to convert $raw"
}
} elseif (Get-Command ffmpeg -ErrorAction SilentlyContinue) {
& ffmpeg -y -f rawvideo -pix_fmt rgba -s "${Width}x${Height}" -i "$raw" "$png" 2>$null
if ($LASTEXITCODE -eq 0) {
Write-Host "Created $png (${Width}x${Height})"
} else {
Write-Host "Failed to convert $raw"
}
} else {
Write-Host "Need 'magick' (ImageMagick) or 'ffmpeg' to convert $raw"
Write-Host " Install: winget install ImageMagick.ImageMagick"
}
}

View file

@ -8,23 +8,32 @@ H=${2:-1024}
echo "Converting debug textures (${W}x${H})..."
for raw in /tmp/wowee_composite_debug.raw /tmp/wowee_equip_composite_debug.raw; do
if [ -f "$raw" ]; then
png="${raw%.raw}.png"
# Try ImageMagick first, fall back to ffmpeg
if command -v convert &>/dev/null; then
convert -size ${W}x${H} -depth 8 rgba:"$raw" "$png" 2>/dev/null && \
echo "Created $png (${W}x${H})" || \
echo "Failed to convert $raw"
elif command -v ffmpeg &>/dev/null; then
ffmpeg -y -f rawvideo -pix_fmt rgba -s ${W}x${H} -i "$raw" "$png" 2>/dev/null && \
echo "Created $png (${W}x${H})" || \
echo "Failed to convert $raw"
else
echo "Need 'convert' (ImageMagick) or 'ffmpeg' to convert $raw"
echo " Install: sudo apt install imagemagick"
fi
TMPD="${TMPDIR:-/tmp}"
# Find raw dumps — filenames include dimensions (e.g. wowee_composite_debug_1024x1024.raw)
shopt -s nullglob
RAW_FILES=("$TMPD"/wowee_*_debug*.raw)
shopt -u nullglob
if [ ${#RAW_FILES[@]} -eq 0 ]; then
echo "No debug dumps found in $TMPD"
echo " (looked for $TMPD/wowee_*_debug*.raw)"
exit 0
fi
for raw in "${RAW_FILES[@]}"; do
png="${raw%.raw}.png"
# Try ImageMagick first, fall back to ffmpeg
if command -v convert &>/dev/null; then
convert -size ${W}x${H} -depth 8 rgba:"$raw" "$png" 2>/dev/null && \
echo "Created $png (${W}x${H})" || \
echo "Failed to convert $raw"
elif command -v ffmpeg &>/dev/null; then
ffmpeg -y -f rawvideo -pix_fmt rgba -s ${W}x${H} -i "$raw" "$png" 2>/dev/null && \
echo "Created $png (${W}x${H})" || \
echo "Failed to convert $raw"
else
echo "Not found: $raw"
echo "Need 'convert' (ImageMagick) or 'ffmpeg' to convert $raw"
echo " Install: sudo apt install imagemagick"
fi
done

3
rebuild.bat Normal file
View file

@ -0,0 +1,3 @@
@echo off
REM Convenience wrapper — launches the PowerShell clean rebuild script.
powershell -ExecutionPolicy Bypass -File "%~dp0rebuild.ps1" %*

50
rebuild.ps1 Normal file
View file

@ -0,0 +1,50 @@
<#
.SYNOPSIS
Clean rebuilds the wowee project (Windows equivalent of rebuild.sh).
.DESCRIPTION
Removes the build directory, reconfigures from scratch, rebuilds, and
creates a directory junction for the Data folder.
#>
$ErrorActionPreference = "Stop"
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
Set-Location $ScriptDir
Write-Host "Clean rebuilding wowee..."
# Remove build directory completely
if (Test-Path "build") {
Write-Host "Removing old build directory..."
Remove-Item -Recurse -Force "build"
}
# Create fresh build directory
New-Item -ItemType Directory -Path "build" | Out-Null
Set-Location "build"
# Configure with CMake
Write-Host "Configuring with CMake..."
& cmake .. -DCMAKE_BUILD_TYPE=Release
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
# Build with all cores
$numProcs = $env:NUMBER_OF_PROCESSORS
if (-not $numProcs) { $numProcs = 4 }
Write-Host "Building with $numProcs cores..."
& cmake --build . --parallel $numProcs
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
# Create Data junction in bin directory
Write-Host "Creating Data junction..."
$binData = Join-Path (Get-Location) "bin\Data"
if (-not (Test-Path $binData)) {
$target = (Resolve-Path (Join-Path (Get-Location) "..\Data")).Path
cmd /c mklink /J "$binData" "$target"
Write-Host " Created Data junction -> $target"
}
Write-Host ""
Write-Host "Clean build complete! Binary: build\bin\wowee.exe"
Write-Host "Run with: cd build\bin && .\wowee.exe"

View file

@ -812,10 +812,10 @@ VkTexture* CharacterRenderer::compositeTextures(const std::vector<std::string>&
}
}
// Debug: dump composite to /tmp for visual inspection
// Debug: dump composite to temp dir for visual inspection
{
std::string dumpPath = "/tmp/wowee_composite_debug_" +
std::to_string(width) + "x" + std::to_string(height) + ".raw";
std::string dumpPath = (std::filesystem::temp_directory_path() / ("wowee_composite_debug_" +
std::to_string(width) + "x" + std::to_string(height) + ".raw")).string();
std::ofstream dump(dumpPath, std::ios::binary);
if (dump) {
dump.write(reinterpret_cast<const char*>(composite.data()),

View file

@ -548,6 +548,12 @@ bool Extractor::enumerateFiles(const Options& opts,
continue;
}
// Verify file actually exists in this archive's hash table
// (listfiles can reference files from other archives)
if (!SFileHasFile(hMpq, fileName.c_str())) {
continue;
}
std::string norm = normalizeWowPath(fileName);
if (opts.onlyUsedDbcs && !wantedDbcs.empty() && !wantedDbcs.contains(norm)) {
continue;
@ -624,24 +630,36 @@ bool Extractor::run(const Options& opts) {
std::atomic<size_t> fileIndex{0};
size_t totalFiles = files.size();
// Open archives ONCE in main thread — StormLib has global state that is not
// thread-safe even with separate handles, so we serialize all MPQ reads.
struct SharedArchive {
HANDLE handle;
int priority;
std::string path;
};
std::vector<SharedArchive> sharedHandles;
for (const auto& ad : archives) {
HANDLE h = nullptr;
if (SFileOpenArchive(ad.path.c_str(), 0, 0, &h)) {
sharedHandles.push_back({h, ad.priority, ad.path});
} else {
std::cerr << " Failed to open archive: " << ad.path << "\n";
}
}
if (sharedHandles.empty()) {
std::cerr << "Failed to open any archives for extraction\n";
return false;
}
if (sharedHandles.size() < archives.size()) {
std::cerr << " Opened " << sharedHandles.size()
<< "/" << archives.size() << " archives\n";
}
// Mutex protecting all StormLib calls (open/read/close are not thread-safe)
std::mutex mpqMutex;
auto workerFn = [&]() {
// Each thread opens ALL archives independently (StormLib is not thread-safe per handle).
// Sorted highest-priority last, so we iterate in reverse to find the winning version.
struct ThreadArchive {
HANDLE handle;
int priority;
};
std::vector<ThreadArchive> threadHandles;
for (const auto& ad : archives) {
HANDLE h = nullptr;
if (SFileOpenArchive(ad.path.c_str(), 0, 0, &h)) {
threadHandles.push_back({h, ad.priority});
}
}
if (threadHandles.empty()) {
std::cerr << "Worker thread: failed to open any archives\n";
return;
}
int failLogCount = 0;
while (true) {
size_t idx = fileIndex.fetch_add(1);
@ -654,35 +672,52 @@ bool Extractor::run(const Options& opts) {
std::string mappedPath = PathMapper::mapPath(wowPath);
std::string fullOutputPath = effectiveOutputDir + "/" + mappedPath;
// Search archives in reverse priority order (highest priority first)
HANDLE hFile = nullptr;
for (auto it = threadHandles.rbegin(); it != threadHandles.rend(); ++it) {
if (SFileOpenFileEx(it->handle, wowPath.c_str(), 0, &hFile)) {
break;
// Read file data from MPQ under lock
std::vector<uint8_t> data;
{
std::lock_guard<std::mutex> lock(mpqMutex);
// Search archives in reverse priority order (highest priority first)
HANDLE hFile = nullptr;
for (auto it = sharedHandles.rbegin(); it != sharedHandles.rend(); ++it) {
if (SFileOpenFileEx(it->handle, wowPath.c_str(), 0, &hFile)) {
break;
}
hFile = nullptr;
}
if (!hFile) {
stats.filesFailed++;
if (failLogCount < 5) {
failLogCount++;
std::cerr << " FAILED open: " << wowPath
<< " (tried " << sharedHandles.size() << " archives)\n";
}
continue;
}
hFile = nullptr;
}
if (!hFile) {
stats.filesFailed++;
continue;
}
DWORD fileSize = SFileGetFileSize(hFile, nullptr);
if (fileSize == SFILE_INVALID_SIZE || fileSize == 0) {
SFileCloseFile(hFile);
stats.filesSkipped++;
continue;
}
DWORD fileSize = SFileGetFileSize(hFile, nullptr);
if (fileSize == SFILE_INVALID_SIZE || fileSize == 0) {
SFileCloseFile(hFile);
stats.filesSkipped++;
continue;
}
std::vector<uint8_t> data(fileSize);
DWORD bytesRead = 0;
if (!SFileReadFile(hFile, data.data(), fileSize, &bytesRead, nullptr)) {
data.resize(fileSize);
DWORD bytesRead = 0;
if (!SFileReadFile(hFile, data.data(), fileSize, &bytesRead, nullptr)) {
SFileCloseFile(hFile);
stats.filesFailed++;
if (failLogCount < 5) {
failLogCount++;
std::cerr << " FAILED read: " << wowPath
<< " (size=" << fileSize << ")\n";
}
continue;
}
SFileCloseFile(hFile);
stats.filesFailed++;
continue;
data.resize(bytesRead);
}
SFileCloseFile(hFile);
data.resize(bytesRead);
// Lock released — CRC computation and disk write happen in parallel
// Compute CRC32
uint32_t crc = ManifestWriter::computeCRC32(data.data(), data.size());
@ -694,6 +729,11 @@ bool Extractor::run(const Options& opts) {
std::ofstream out(fullOutputPath, std::ios::binary);
if (!out.is_open()) {
stats.filesFailed++;
if (failLogCount < 5) {
failLogCount++;
std::lock_guard<std::mutex> lock(manifestMutex);
std::cerr << " FAILED write: " << fullOutputPath << "\n";
}
continue;
}
out.write(reinterpret_cast<const char*>(data.data()), data.size());
@ -721,10 +761,6 @@ bool Extractor::run(const Options& opts) {
<< std::flush;
}
}
for (auto& th : threadHandles) {
SFileCloseArchive(th.handle);
}
};
std::cout << "Extracting " << totalFiles << " files using " << numThreads << " threads...\n";
@ -737,10 +773,30 @@ bool Extractor::run(const Options& opts) {
t.join();
}
std::cout << "\r Extracted " << stats.filesExtracted.load() << " files ("
// Close archives (opened once in main thread)
for (auto& sh : sharedHandles) {
SFileCloseArchive(sh.handle);
}
auto extracted = stats.filesExtracted.load();
auto failed = stats.filesFailed.load();
auto skipped = stats.filesSkipped.load();
std::cout << "\n Extracted " << extracted << " files ("
<< stats.bytesExtracted.load() / (1024 * 1024) << " MB), "
<< stats.filesSkipped.load() << " skipped, "
<< stats.filesFailed.load() << " failed\n";
<< skipped << " skipped, "
<< failed << " failed\n";
// If most files failed, print a diagnostic hint
if (failed > 0 && failed > extracted * 10) {
std::cerr << "\nWARNING: " << failed << " out of " << totalFiles
<< " files failed to extract.\n"
<< " This usually means worker threads could not open one or more MPQ archives.\n"
<< " Common causes:\n"
<< " - MPQ files on a network/external drive with access restrictions\n"
<< " - Another program (WoW client, antivirus) has the MPQ files locked\n"
<< " - Too many threads for the OS file-handle limit (try --threads 1)\n"
<< " Re-run with --verbose for detailed diagnostics.\n";
}
// Merge with existing manifest so partial extractions don't nuke prior entries
std::string manifestPath = effectiveOutputDir + "/manifest.json";