mirror of
https://github.com/Kelsidavis/WoWee.git
synced 2026-03-22 15:20:15 +00:00
Add Windows build scripts, fix multi-threaded MPQ extraction, and cross-platform temp paths
- Add build.ps1/bat, rebuild.ps1/bat, debug_texture.ps1/bat (Windows equivalents of existing bash scripts, using directory junctions for Data link) - Fix asset extractor: StormLib is not thread-safe even with separate handles per thread. Serialize all MPQ reads behind a mutex while keeping CRC computation and disk writes parallel. Previously caused 99.8% extraction failures with >1 thread. - Add SFileHasFile() check during enumeration to skip listfile-only entries - Add diagnostic logging for extraction failures (first 5 per thread + summary) - Use std::filesystem::temp_directory_path() instead of hardcoded /tmp/ in character_renderer.cpp debug dumps - Update debug_texture.sh to use $TMPDIR fallback and glob for actual dump filenames
This commit is contained in:
parent
570dec8b88
commit
1fab17e639
9 changed files with 300 additions and 68 deletions
3
build.bat
Normal file
3
build.bat
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
@echo off
|
||||
REM Convenience wrapper — launches the PowerShell build script.
|
||||
powershell -ExecutionPolicy Bypass -File "%~dp0build.ps1" %*
|
||||
44
build.ps1
Normal file
44
build.ps1
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Builds the wowee project (Windows equivalent of build.sh).
|
||||
|
||||
.DESCRIPTION
|
||||
Creates a build directory, runs CMake configure + build, and creates a
|
||||
directory junction for the Data folder so the binary can find assets.
|
||||
#>
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
Set-Location $ScriptDir
|
||||
|
||||
Write-Host "Building wowee..."
|
||||
|
||||
# Create build directory if it doesn't exist
|
||||
if (-not (Test-Path "build")) {
|
||||
New-Item -ItemType Directory -Path "build" | Out-Null
|
||||
}
|
||||
Set-Location "build"
|
||||
|
||||
# Configure with CMake
|
||||
Write-Host "Configuring with CMake..."
|
||||
& cmake .. -DCMAKE_BUILD_TYPE=Release
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
|
||||
# Build with all cores
|
||||
$numProcs = $env:NUMBER_OF_PROCESSORS
|
||||
if (-not $numProcs) { $numProcs = 4 }
|
||||
Write-Host "Building with $numProcs cores..."
|
||||
& cmake --build . --parallel $numProcs
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
|
||||
# Ensure Data junction exists in bin directory
|
||||
$binData = Join-Path (Get-Location) "bin\Data"
|
||||
if (-not (Test-Path $binData)) {
|
||||
$target = (Resolve-Path (Join-Path (Get-Location) "..\Data")).Path
|
||||
cmd /c mklink /J "$binData" "$target"
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Build complete! Binary: build\bin\wowee.exe"
|
||||
Write-Host "Run with: cd build\bin && .\wowee.exe"
|
||||
3
debug_texture.bat
Normal file
3
debug_texture.bat
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
@echo off
|
||||
REM Convenience wrapper — launches the PowerShell texture debug script.
|
||||
powershell -ExecutionPolicy Bypass -File "%~dp0debug_texture.ps1" %*
|
||||
64
debug_texture.ps1
Normal file
64
debug_texture.ps1
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Converts raw RGBA texture dumps to PNG for visual inspection (Windows equivalent of debug_texture.sh).
|
||||
|
||||
.PARAMETER Width
|
||||
Texture width in pixels. Defaults to 1024.
|
||||
|
||||
.PARAMETER Height
|
||||
Texture height in pixels. Defaults to 1024.
|
||||
|
||||
.EXAMPLE
|
||||
.\debug_texture.ps1
|
||||
.\debug_texture.ps1 -Width 2048 -Height 2048
|
||||
#>
|
||||
|
||||
param(
|
||||
[int]$Width = 1024,
|
||||
[int]$Height = 1024
|
||||
)
|
||||
|
||||
$TempDir = $env:TEMP
|
||||
|
||||
Write-Host "Converting debug textures (${Width}x${Height})..."
|
||||
|
||||
# Find raw dumps — filenames include dimensions (e.g. wowee_composite_debug_1024x1024.raw)
|
||||
$rawFiles = Get-ChildItem -Path $TempDir -Filter "wowee_*_debug*.raw" -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $rawFiles) {
|
||||
Write-Host "No debug dumps found in $TempDir"
|
||||
Write-Host " (looked for $TempDir\wowee_*_debug*.raw)"
|
||||
exit 0
|
||||
}
|
||||
|
||||
foreach ($rawItem in $rawFiles) {
|
||||
$raw = $rawItem.FullName
|
||||
$png = $raw -replace '\.raw$', '.png'
|
||||
|
||||
# Try ImageMagick first, fall back to ffmpeg
|
||||
if (Get-Command magick -ErrorAction SilentlyContinue) {
|
||||
& magick -size "${Width}x${Height}" -depth 8 "rgba:$raw" "$png" 2>$null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host "Created $png (${Width}x${Height})"
|
||||
} else {
|
||||
Write-Host "Failed to convert $raw"
|
||||
}
|
||||
} elseif (Get-Command convert -ErrorAction SilentlyContinue) {
|
||||
& convert -size "${Width}x${Height}" -depth 8 "rgba:$raw" "$png" 2>$null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host "Created $png (${Width}x${Height})"
|
||||
} else {
|
||||
Write-Host "Failed to convert $raw"
|
||||
}
|
||||
} elseif (Get-Command ffmpeg -ErrorAction SilentlyContinue) {
|
||||
& ffmpeg -y -f rawvideo -pix_fmt rgba -s "${Width}x${Height}" -i "$raw" "$png" 2>$null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host "Created $png (${Width}x${Height})"
|
||||
} else {
|
||||
Write-Host "Failed to convert $raw"
|
||||
}
|
||||
} else {
|
||||
Write-Host "Need 'magick' (ImageMagick) or 'ffmpeg' to convert $raw"
|
||||
Write-Host " Install: winget install ImageMagick.ImageMagick"
|
||||
}
|
||||
}
|
||||
|
|
@ -8,23 +8,32 @@ H=${2:-1024}
|
|||
|
||||
echo "Converting debug textures (${W}x${H})..."
|
||||
|
||||
for raw in /tmp/wowee_composite_debug.raw /tmp/wowee_equip_composite_debug.raw; do
|
||||
if [ -f "$raw" ]; then
|
||||
png="${raw%.raw}.png"
|
||||
# Try ImageMagick first, fall back to ffmpeg
|
||||
if command -v convert &>/dev/null; then
|
||||
convert -size ${W}x${H} -depth 8 rgba:"$raw" "$png" 2>/dev/null && \
|
||||
echo "Created $png (${W}x${H})" || \
|
||||
echo "Failed to convert $raw"
|
||||
elif command -v ffmpeg &>/dev/null; then
|
||||
ffmpeg -y -f rawvideo -pix_fmt rgba -s ${W}x${H} -i "$raw" "$png" 2>/dev/null && \
|
||||
echo "Created $png (${W}x${H})" || \
|
||||
echo "Failed to convert $raw"
|
||||
else
|
||||
echo "Need 'convert' (ImageMagick) or 'ffmpeg' to convert $raw"
|
||||
echo " Install: sudo apt install imagemagick"
|
||||
fi
|
||||
TMPD="${TMPDIR:-/tmp}"
|
||||
|
||||
# Find raw dumps — filenames include dimensions (e.g. wowee_composite_debug_1024x1024.raw)
|
||||
shopt -s nullglob
|
||||
RAW_FILES=("$TMPD"/wowee_*_debug*.raw)
|
||||
shopt -u nullglob
|
||||
|
||||
if [ ${#RAW_FILES[@]} -eq 0 ]; then
|
||||
echo "No debug dumps found in $TMPD"
|
||||
echo " (looked for $TMPD/wowee_*_debug*.raw)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for raw in "${RAW_FILES[@]}"; do
|
||||
png="${raw%.raw}.png"
|
||||
# Try ImageMagick first, fall back to ffmpeg
|
||||
if command -v convert &>/dev/null; then
|
||||
convert -size ${W}x${H} -depth 8 rgba:"$raw" "$png" 2>/dev/null && \
|
||||
echo "Created $png (${W}x${H})" || \
|
||||
echo "Failed to convert $raw"
|
||||
elif command -v ffmpeg &>/dev/null; then
|
||||
ffmpeg -y -f rawvideo -pix_fmt rgba -s ${W}x${H} -i "$raw" "$png" 2>/dev/null && \
|
||||
echo "Created $png (${W}x${H})" || \
|
||||
echo "Failed to convert $raw"
|
||||
else
|
||||
echo "Not found: $raw"
|
||||
echo "Need 'convert' (ImageMagick) or 'ffmpeg' to convert $raw"
|
||||
echo " Install: sudo apt install imagemagick"
|
||||
fi
|
||||
done
|
||||
|
|
|
|||
3
rebuild.bat
Normal file
3
rebuild.bat
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
@echo off
|
||||
REM Convenience wrapper — launches the PowerShell clean rebuild script.
|
||||
powershell -ExecutionPolicy Bypass -File "%~dp0rebuild.ps1" %*
|
||||
50
rebuild.ps1
Normal file
50
rebuild.ps1
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Clean rebuilds the wowee project (Windows equivalent of rebuild.sh).
|
||||
|
||||
.DESCRIPTION
|
||||
Removes the build directory, reconfigures from scratch, rebuilds, and
|
||||
creates a directory junction for the Data folder.
|
||||
#>
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
Set-Location $ScriptDir
|
||||
|
||||
Write-Host "Clean rebuilding wowee..."
|
||||
|
||||
# Remove build directory completely
|
||||
if (Test-Path "build") {
|
||||
Write-Host "Removing old build directory..."
|
||||
Remove-Item -Recurse -Force "build"
|
||||
}
|
||||
|
||||
# Create fresh build directory
|
||||
New-Item -ItemType Directory -Path "build" | Out-Null
|
||||
Set-Location "build"
|
||||
|
||||
# Configure with CMake
|
||||
Write-Host "Configuring with CMake..."
|
||||
& cmake .. -DCMAKE_BUILD_TYPE=Release
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
|
||||
# Build with all cores
|
||||
$numProcs = $env:NUMBER_OF_PROCESSORS
|
||||
if (-not $numProcs) { $numProcs = 4 }
|
||||
Write-Host "Building with $numProcs cores..."
|
||||
& cmake --build . --parallel $numProcs
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
|
||||
# Create Data junction in bin directory
|
||||
Write-Host "Creating Data junction..."
|
||||
$binData = Join-Path (Get-Location) "bin\Data"
|
||||
if (-not (Test-Path $binData)) {
|
||||
$target = (Resolve-Path (Join-Path (Get-Location) "..\Data")).Path
|
||||
cmd /c mklink /J "$binData" "$target"
|
||||
Write-Host " Created Data junction -> $target"
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Clean build complete! Binary: build\bin\wowee.exe"
|
||||
Write-Host "Run with: cd build\bin && .\wowee.exe"
|
||||
|
|
@ -812,10 +812,10 @@ VkTexture* CharacterRenderer::compositeTextures(const std::vector<std::string>&
|
|||
}
|
||||
}
|
||||
|
||||
// Debug: dump composite to /tmp for visual inspection
|
||||
// Debug: dump composite to temp dir for visual inspection
|
||||
{
|
||||
std::string dumpPath = "/tmp/wowee_composite_debug_" +
|
||||
std::to_string(width) + "x" + std::to_string(height) + ".raw";
|
||||
std::string dumpPath = (std::filesystem::temp_directory_path() / ("wowee_composite_debug_" +
|
||||
std::to_string(width) + "x" + std::to_string(height) + ".raw")).string();
|
||||
std::ofstream dump(dumpPath, std::ios::binary);
|
||||
if (dump) {
|
||||
dump.write(reinterpret_cast<const char*>(composite.data()),
|
||||
|
|
|
|||
|
|
@ -548,6 +548,12 @@ bool Extractor::enumerateFiles(const Options& opts,
|
|||
continue;
|
||||
}
|
||||
|
||||
// Verify file actually exists in this archive's hash table
|
||||
// (listfiles can reference files from other archives)
|
||||
if (!SFileHasFile(hMpq, fileName.c_str())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string norm = normalizeWowPath(fileName);
|
||||
if (opts.onlyUsedDbcs && !wantedDbcs.empty() && !wantedDbcs.contains(norm)) {
|
||||
continue;
|
||||
|
|
@ -624,24 +630,36 @@ bool Extractor::run(const Options& opts) {
|
|||
std::atomic<size_t> fileIndex{0};
|
||||
size_t totalFiles = files.size();
|
||||
|
||||
// Open archives ONCE in main thread — StormLib has global state that is not
|
||||
// thread-safe even with separate handles, so we serialize all MPQ reads.
|
||||
struct SharedArchive {
|
||||
HANDLE handle;
|
||||
int priority;
|
||||
std::string path;
|
||||
};
|
||||
std::vector<SharedArchive> sharedHandles;
|
||||
for (const auto& ad : archives) {
|
||||
HANDLE h = nullptr;
|
||||
if (SFileOpenArchive(ad.path.c_str(), 0, 0, &h)) {
|
||||
sharedHandles.push_back({h, ad.priority, ad.path});
|
||||
} else {
|
||||
std::cerr << " Failed to open archive: " << ad.path << "\n";
|
||||
}
|
||||
}
|
||||
if (sharedHandles.empty()) {
|
||||
std::cerr << "Failed to open any archives for extraction\n";
|
||||
return false;
|
||||
}
|
||||
if (sharedHandles.size() < archives.size()) {
|
||||
std::cerr << " Opened " << sharedHandles.size()
|
||||
<< "/" << archives.size() << " archives\n";
|
||||
}
|
||||
|
||||
// Mutex protecting all StormLib calls (open/read/close are not thread-safe)
|
||||
std::mutex mpqMutex;
|
||||
|
||||
auto workerFn = [&]() {
|
||||
// Each thread opens ALL archives independently (StormLib is not thread-safe per handle).
|
||||
// Sorted highest-priority last, so we iterate in reverse to find the winning version.
|
||||
struct ThreadArchive {
|
||||
HANDLE handle;
|
||||
int priority;
|
||||
};
|
||||
std::vector<ThreadArchive> threadHandles;
|
||||
for (const auto& ad : archives) {
|
||||
HANDLE h = nullptr;
|
||||
if (SFileOpenArchive(ad.path.c_str(), 0, 0, &h)) {
|
||||
threadHandles.push_back({h, ad.priority});
|
||||
}
|
||||
}
|
||||
if (threadHandles.empty()) {
|
||||
std::cerr << "Worker thread: failed to open any archives\n";
|
||||
return;
|
||||
}
|
||||
int failLogCount = 0;
|
||||
|
||||
while (true) {
|
||||
size_t idx = fileIndex.fetch_add(1);
|
||||
|
|
@ -654,35 +672,52 @@ bool Extractor::run(const Options& opts) {
|
|||
std::string mappedPath = PathMapper::mapPath(wowPath);
|
||||
std::string fullOutputPath = effectiveOutputDir + "/" + mappedPath;
|
||||
|
||||
// Search archives in reverse priority order (highest priority first)
|
||||
HANDLE hFile = nullptr;
|
||||
for (auto it = threadHandles.rbegin(); it != threadHandles.rend(); ++it) {
|
||||
if (SFileOpenFileEx(it->handle, wowPath.c_str(), 0, &hFile)) {
|
||||
break;
|
||||
// Read file data from MPQ under lock
|
||||
std::vector<uint8_t> data;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mpqMutex);
|
||||
|
||||
// Search archives in reverse priority order (highest priority first)
|
||||
HANDLE hFile = nullptr;
|
||||
for (auto it = sharedHandles.rbegin(); it != sharedHandles.rend(); ++it) {
|
||||
if (SFileOpenFileEx(it->handle, wowPath.c_str(), 0, &hFile)) {
|
||||
break;
|
||||
}
|
||||
hFile = nullptr;
|
||||
}
|
||||
if (!hFile) {
|
||||
stats.filesFailed++;
|
||||
if (failLogCount < 5) {
|
||||
failLogCount++;
|
||||
std::cerr << " FAILED open: " << wowPath
|
||||
<< " (tried " << sharedHandles.size() << " archives)\n";
|
||||
}
|
||||
continue;
|
||||
}
|
||||
hFile = nullptr;
|
||||
}
|
||||
if (!hFile) {
|
||||
stats.filesFailed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
DWORD fileSize = SFileGetFileSize(hFile, nullptr);
|
||||
if (fileSize == SFILE_INVALID_SIZE || fileSize == 0) {
|
||||
SFileCloseFile(hFile);
|
||||
stats.filesSkipped++;
|
||||
continue;
|
||||
}
|
||||
DWORD fileSize = SFileGetFileSize(hFile, nullptr);
|
||||
if (fileSize == SFILE_INVALID_SIZE || fileSize == 0) {
|
||||
SFileCloseFile(hFile);
|
||||
stats.filesSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> data(fileSize);
|
||||
DWORD bytesRead = 0;
|
||||
if (!SFileReadFile(hFile, data.data(), fileSize, &bytesRead, nullptr)) {
|
||||
data.resize(fileSize);
|
||||
DWORD bytesRead = 0;
|
||||
if (!SFileReadFile(hFile, data.data(), fileSize, &bytesRead, nullptr)) {
|
||||
SFileCloseFile(hFile);
|
||||
stats.filesFailed++;
|
||||
if (failLogCount < 5) {
|
||||
failLogCount++;
|
||||
std::cerr << " FAILED read: " << wowPath
|
||||
<< " (size=" << fileSize << ")\n";
|
||||
}
|
||||
continue;
|
||||
}
|
||||
SFileCloseFile(hFile);
|
||||
stats.filesFailed++;
|
||||
continue;
|
||||
data.resize(bytesRead);
|
||||
}
|
||||
SFileCloseFile(hFile);
|
||||
data.resize(bytesRead);
|
||||
// Lock released — CRC computation and disk write happen in parallel
|
||||
|
||||
// Compute CRC32
|
||||
uint32_t crc = ManifestWriter::computeCRC32(data.data(), data.size());
|
||||
|
|
@ -694,6 +729,11 @@ bool Extractor::run(const Options& opts) {
|
|||
std::ofstream out(fullOutputPath, std::ios::binary);
|
||||
if (!out.is_open()) {
|
||||
stats.filesFailed++;
|
||||
if (failLogCount < 5) {
|
||||
failLogCount++;
|
||||
std::lock_guard<std::mutex> lock(manifestMutex);
|
||||
std::cerr << " FAILED write: " << fullOutputPath << "\n";
|
||||
}
|
||||
continue;
|
||||
}
|
||||
out.write(reinterpret_cast<const char*>(data.data()), data.size());
|
||||
|
|
@ -721,10 +761,6 @@ bool Extractor::run(const Options& opts) {
|
|||
<< std::flush;
|
||||
}
|
||||
}
|
||||
|
||||
for (auto& th : threadHandles) {
|
||||
SFileCloseArchive(th.handle);
|
||||
}
|
||||
};
|
||||
|
||||
std::cout << "Extracting " << totalFiles << " files using " << numThreads << " threads...\n";
|
||||
|
|
@ -737,10 +773,30 @@ bool Extractor::run(const Options& opts) {
|
|||
t.join();
|
||||
}
|
||||
|
||||
std::cout << "\r Extracted " << stats.filesExtracted.load() << " files ("
|
||||
// Close archives (opened once in main thread)
|
||||
for (auto& sh : sharedHandles) {
|
||||
SFileCloseArchive(sh.handle);
|
||||
}
|
||||
|
||||
auto extracted = stats.filesExtracted.load();
|
||||
auto failed = stats.filesFailed.load();
|
||||
auto skipped = stats.filesSkipped.load();
|
||||
std::cout << "\n Extracted " << extracted << " files ("
|
||||
<< stats.bytesExtracted.load() / (1024 * 1024) << " MB), "
|
||||
<< stats.filesSkipped.load() << " skipped, "
|
||||
<< stats.filesFailed.load() << " failed\n";
|
||||
<< skipped << " skipped, "
|
||||
<< failed << " failed\n";
|
||||
|
||||
// If most files failed, print a diagnostic hint
|
||||
if (failed > 0 && failed > extracted * 10) {
|
||||
std::cerr << "\nWARNING: " << failed << " out of " << totalFiles
|
||||
<< " files failed to extract.\n"
|
||||
<< " This usually means worker threads could not open one or more MPQ archives.\n"
|
||||
<< " Common causes:\n"
|
||||
<< " - MPQ files on a network/external drive with access restrictions\n"
|
||||
<< " - Another program (WoW client, antivirus) has the MPQ files locked\n"
|
||||
<< " - Too many threads for the OS file-handle limit (try --threads 1)\n"
|
||||
<< " Re-run with --verbose for detailed diagnostics.\n";
|
||||
}
|
||||
|
||||
// Merge with existing manifest so partial extractions don't nuke prior entries
|
||||
std::string manifestPath = effectiveOutputDir + "/manifest.json";
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue