From aa16a687c2dff73dd03946a9fcd18d401ccc19c9 Mon Sep 17 00:00:00 2001 From: Kelsi Date: Thu, 12 Feb 2026 20:32:14 -0800 Subject: [PATCH] Replace MPQ runtime with loose file asset system Extract assets from MPQ archives into organized loose files indexed by manifest.json, enabling fully parallel reads without StormLib serialization. Add asset_extract and blp_convert tools, PNG texture override support. --- CMakeLists.txt | 62 +++- include/pipeline/asset_manager.hpp | 37 ++- include/pipeline/asset_manifest.hpp | 74 +++++ include/pipeline/loose_file_reader.hpp | 40 +++ src/pipeline/asset_manager.cpp | 151 +++++----- src/pipeline/asset_manifest.cpp | 97 ++++++ src/pipeline/loose_file_reader.cpp | 47 +++ src/rendering/minimap.cpp | 2 +- tools/asset_extract/extractor.cpp | 380 ++++++++++++++++++++++++ tools/asset_extract/extractor.hpp | 43 +++ tools/asset_extract/main.cpp | 62 ++++ tools/asset_extract/manifest_writer.cpp | 67 +++++ tools/asset_extract/manifest_writer.hpp | 40 +++ tools/asset_extract/path_mapper.cpp | 207 +++++++++++++ tools/asset_extract/path_mapper.hpp | 32 ++ tools/blp_convert/main.cpp | 187 ++++++++++++ 16 files changed, 1427 insertions(+), 101 deletions(-) create mode 100644 include/pipeline/asset_manifest.hpp create mode 100644 include/pipeline/loose_file_reader.hpp create mode 100644 src/pipeline/asset_manifest.cpp create mode 100644 src/pipeline/loose_file_reader.cpp create mode 100644 tools/asset_extract/extractor.cpp create mode 100644 tools/asset_extract/extractor.hpp create mode 100644 tools/asset_extract/main.cpp create mode 100644 tools/asset_extract/manifest_writer.cpp create mode 100644 tools/asset_extract/manifest_writer.hpp create mode 100644 tools/asset_extract/path_mapper.cpp create mode 100644 tools/asset_extract/path_mapper.hpp create mode 100644 tools/blp_convert/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a4ec9ec8..8813334b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,15 +43,10 @@ if(NOT glm_FOUND) message(STATUS "GLM not found, will use system includes or download") endif() -# StormLib for MPQ archives +# StormLib for MPQ extraction tool (not needed for main executable) find_library(STORMLIB_LIBRARY NAMES StormLib stormlib storm) find_path(STORMLIB_INCLUDE_DIR StormLib.h PATH_SUFFIXES StormLib) -if(NOT STORMLIB_LIBRARY OR NOT STORMLIB_INCLUDE_DIR) - message(WARNING "StormLib not found. You may need to build it manually.") - message(WARNING "Get it from: https://github.com/ladislav-zezula/StormLib") -endif() - # Include ImGui as a static library (we'll add the sources) set(IMGUI_DIR ${CMAKE_CURRENT_SOURCE_DIR}/extern/imgui) if(EXISTS ${IMGUI_DIR}) @@ -128,10 +123,11 @@ set(WOWEE_SOURCES src/audio/movement_sound_manager.cpp # Pipeline (asset loaders) - src/pipeline/mpq_manager.cpp src/pipeline/blp_loader.cpp src/pipeline/dbc_loader.cpp src/pipeline/asset_manager.cpp + src/pipeline/asset_manifest.cpp + src/pipeline/loose_file_reader.cpp src/pipeline/m2_loader.cpp src/pipeline/wmo_loader.cpp src/pipeline/adt_loader.cpp @@ -233,8 +229,9 @@ set(WOWEE_HEADERS include/audio/spell_sound_manager.hpp include/audio/movement_sound_manager.hpp - include/pipeline/mpq_manager.hpp include/pipeline/blp_loader.hpp + include/pipeline/asset_manifest.hpp + include/pipeline/loose_file_reader.hpp include/pipeline/m2_loader.hpp include/pipeline/wmo_loader.hpp include/pipeline/adt_loader.hpp @@ -325,13 +322,6 @@ if(WIN32) endif() endif() -# Link StormLib if found -if(STORMLIB_LIBRARY AND STORMLIB_INCLUDE_DIR) - target_link_libraries(wowee PRIVATE ${STORMLIB_LIBRARY}) - target_include_directories(wowee PRIVATE ${STORMLIB_INCLUDE_DIR}) - target_compile_definitions(wowee PRIVATE HAVE_STORMLIB) -endif() - # Link ImGui if available if(TARGET imgui) target_link_libraries(wowee PRIVATE imgui) @@ -385,6 +375,47 @@ if(UNIX AND NOT APPLE) RENAME wowee.png) endif() +# ---- Tool: asset_extract (MPQ → loose files) ---- +if(STORMLIB_LIBRARY AND STORMLIB_INCLUDE_DIR) + add_executable(asset_extract + tools/asset_extract/main.cpp + tools/asset_extract/extractor.cpp + tools/asset_extract/path_mapper.cpp + tools/asset_extract/manifest_writer.cpp + ) + target_include_directories(asset_extract PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/tools/asset_extract + ${STORMLIB_INCLUDE_DIR} + ) + target_link_libraries(asset_extract PRIVATE + ${STORMLIB_LIBRARY} + ZLIB::ZLIB + Threads::Threads + ) + set_target_properties(asset_extract PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin + ) + message(STATUS " asset_extract tool: ENABLED") +else() + message(STATUS " asset_extract tool: DISABLED (requires StormLib)") +endif() + +# ---- Tool: blp_convert (BLP ↔ PNG) ---- +add_executable(blp_convert + tools/blp_convert/main.cpp + src/pipeline/blp_loader.cpp + src/core/logger.cpp +) +target_include_directories(blp_convert PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/extern +) +target_link_libraries(blp_convert PRIVATE Threads::Threads) +set_target_properties(blp_convert PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin +) + # Print configuration summary message(STATUS "") message(STATUS "Wowee Configuration:") @@ -392,6 +423,5 @@ message(STATUS " C++ Standard: ${CMAKE_CXX_STANDARD}") message(STATUS " Build Type: ${CMAKE_BUILD_TYPE}") message(STATUS " SDL2: ${SDL2_VERSION}") message(STATUS " OpenSSL: ${OPENSSL_VERSION}") -message(STATUS " StormLib: ${STORMLIB_LIBRARY}") message(STATUS " ImGui: ${IMGUI_DIR}") message(STATUS "") diff --git a/include/pipeline/asset_manager.hpp b/include/pipeline/asset_manager.hpp index ba69b75d..b146bd8e 100644 --- a/include/pipeline/asset_manager.hpp +++ b/include/pipeline/asset_manager.hpp @@ -1,8 +1,9 @@ #pragma once -#include "pipeline/mpq_manager.hpp" #include "pipeline/blp_loader.hpp" #include "pipeline/dbc_loader.hpp" +#include "pipeline/asset_manifest.hpp" +#include "pipeline/loose_file_reader.hpp" #include #include #include @@ -14,7 +15,9 @@ namespace pipeline { /** * AssetManager - Unified interface for loading WoW assets * - * Coordinates MPQ archives, texture loading, and database files + * Reads pre-extracted loose files indexed by manifest.json. + * Use the asset_extract tool to extract MPQ archives first. + * All reads are fully parallel (no serialization mutex needed). */ class AssetManager { public: @@ -23,7 +26,7 @@ public: /** * Initialize asset manager - * @param dataPath Path to WoW Data directory + * @param dataPath Path to directory containing manifest.json and extracted assets * @return true if initialization succeeded */ bool initialize(const std::string& dataPath); @@ -60,33 +63,27 @@ public: std::shared_ptr getDBC(const std::string& name) const; /** - * Check if a file exists in MPQ archives + * Check if a file exists * @param path Virtual file path * @return true if file exists */ bool fileExists(const std::string& path) const; /** - * Read raw file data from MPQ archives + * Read raw file data * @param path Virtual file path * @return File contents (empty if not found) */ std::vector readFile(const std::string& path) const; /** - * Read optional file data from MPQ archives without warning spam. + * Read optional file data without warning spam. * Intended for probe-style lookups (e.g. external .anim variants). * @param path Virtual file path * @return File contents (empty if not found) */ std::vector readFileOptional(const std::string& path) const; - /** - * Get MPQ manager for direct access - */ - MPQManager& getMPQManager() { return mpqManager; } - const MPQManager& getMPQManager() const { return mpqManager; } - /** * Get loaded DBC count */ @@ -108,12 +105,14 @@ private: bool initialized = false; std::string dataPath; - MPQManager mpqManager; - mutable std::mutex readMutex; + // Loose file backend + AssetManifest manifest_; + LooseFileReader looseReader_; + mutable std::mutex cacheMutex; std::map> dbcCache; - // Decompressed file cache (LRU, dynamic budget based on system RAM) + // File cache (LRU, dynamic budget based on system RAM) struct CachedFile { std::vector data; uint64_t lastAccessTime; @@ -125,6 +124,14 @@ private: mutable size_t fileCacheMisses = 0; mutable size_t fileCacheBudget = 1024 * 1024 * 1024; // Dynamic, starts at 1GB + void setupFileCacheBudget(); + + /** + * Try to load a PNG override for a BLP path. + * Returns valid BLPImage if PNG found, invalid otherwise. + */ + BLPImage tryLoadPngOverride(const std::string& normalizedPath) const; + /** * Normalize path for case-insensitive lookup */ diff --git a/include/pipeline/asset_manifest.hpp b/include/pipeline/asset_manifest.hpp new file mode 100644 index 00000000..4f658646 --- /dev/null +++ b/include/pipeline/asset_manifest.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include +#include +#include +#include + +namespace wowee { +namespace pipeline { + +/** + * AssetManifest - Maps WoW virtual paths to filesystem paths + * + * Loaded once at startup from manifest.json. Read-only after init, + * so concurrent reads are safe without a mutex. + */ +class AssetManifest { +public: + struct Entry { + std::string filesystemPath; // Relative path from basePath (forward slashes) + uint64_t size; // File size in bytes + uint32_t crc32; // CRC32 for integrity verification + }; + + AssetManifest() = default; + + /** + * Load manifest from JSON file + * @param manifestPath Full path to manifest.json + * @return true if loaded successfully + */ + bool load(const std::string& manifestPath); + + /** + * Lookup an entry by normalized WoW path (lowercase, backslash) + * @return Pointer to entry or nullptr if not found + */ + const Entry* lookup(const std::string& normalizedWowPath) const; + + /** + * Resolve full filesystem path for a WoW virtual path + * @return Full filesystem path or empty string if not found + */ + std::string resolveFilesystemPath(const std::string& normalizedWowPath) const; + + /** + * Check if an entry exists + */ + bool hasEntry(const std::string& normalizedWowPath) const; + + /** + * Get base path (directory containing extracted assets) + */ + const std::string& getBasePath() const { return basePath_; } + + /** + * Get total number of entries + */ + size_t getEntryCount() const { return entries_.size(); } + + /** + * Check if manifest is loaded + */ + bool isLoaded() const { return loaded_; } + +private: + bool loaded_ = false; + std::string basePath_; // Root directory for extracted assets + std::string manifestDir_; // Directory containing manifest.json + std::unordered_map entries_; +}; + +} // namespace pipeline +} // namespace wowee diff --git a/include/pipeline/loose_file_reader.hpp b/include/pipeline/loose_file_reader.hpp new file mode 100644 index 00000000..95733aec --- /dev/null +++ b/include/pipeline/loose_file_reader.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +namespace wowee { +namespace pipeline { + +/** + * LooseFileReader - Thread-safe filesystem file reader + * + * Each read opens its own file descriptor, so no mutex is needed. + * This replaces the serialized MPQ read path. + */ +class LooseFileReader { +public: + LooseFileReader() = default; + + /** + * Read entire file into memory + * @param filesystemPath Full path to file on disk + * @return File contents (empty if not found or error) + */ + static std::vector readFile(const std::string& filesystemPath); + + /** + * Check if a file exists on disk + */ + static bool fileExists(const std::string& filesystemPath); + + /** + * Get file size without reading + * @return Size in bytes, or 0 if not found + */ + static uint64_t getFileSize(const std::string& filesystemPath); +}; + +} // namespace pipeline +} // namespace wowee diff --git a/src/pipeline/asset_manager.cpp b/src/pipeline/asset_manager.cpp index 08034e1c..9a9cc5dc 100644 --- a/src/pipeline/asset_manager.cpp +++ b/src/pipeline/asset_manager.cpp @@ -3,8 +3,11 @@ #include "core/memory_monitor.hpp" #include #include +#include #include +#include "stb_image.h" + namespace wowee { namespace pipeline { @@ -40,26 +43,36 @@ bool AssetManager::initialize(const std::string& dataPath_) { dataPath = dataPath_; LOG_INFO("Initializing asset manager with data path: ", dataPath); - // Initialize MPQ manager - if (!mpqManager.initialize(dataPath)) { - LOG_ERROR("Failed to initialize MPQ manager"); + setupFileCacheBudget(); + + std::string manifestPath = dataPath + "/manifest.json"; + if (!std::filesystem::exists(manifestPath)) { + LOG_ERROR("manifest.json not found in: ", dataPath); + LOG_ERROR("Run asset_extract to extract MPQ archives first"); return false; } - // Set dynamic file cache budget based on available RAM. - // Bias toward MPQ decompressed-file caching to minimize runtime read/decompress stalls. + if (!manifest_.load(manifestPath)) { + LOG_ERROR("Failed to load manifest"); + return false; + } + + initialized = true; + LOG_INFO("Asset manager initialized: ", manifest_.getEntryCount(), + " files indexed (file cache: ", fileCacheBudget / (1024 * 1024), " MB)"); + return true; +} + +void AssetManager::setupFileCacheBudget() { auto& memMonitor = core::MemoryMonitor::getInstance(); size_t recommendedBudget = memMonitor.getRecommendedCacheBudget(); - size_t dynamicBudget = (recommendedBudget * 3) / 4; // 75% of global cache budget + size_t dynamicBudget = (recommendedBudget * 3) / 4; - // The MemoryMonitor recommendation is intentionally aggressive; without a cap, large patch MPQs - // can cause the decompressed-file cache to balloon into tens/hundreds of GB and push the OS into swap. - // Provide env overrides and clamp to a safe default. const size_t envFixedMB = parseEnvSizeMB("WOWEE_FILE_CACHE_MB"); const size_t envMaxMB = parseEnvSizeMB("WOWEE_FILE_CACHE_MAX_MB"); - const size_t minBudgetBytes = 256ull * 1024ull * 1024ull; // 256 MB - const size_t defaultMaxBudgetBytes = 32768ull * 1024ull * 1024ull; // 32 GB + const size_t minBudgetBytes = 256ull * 1024ull * 1024ull; + const size_t defaultMaxBudgetBytes = 32768ull * 1024ull * 1024ull; const size_t maxBudgetBytes = (envMaxMB > 0) ? (envMaxMB * 1024ull * 1024ull) : defaultMaxBudgetBytes; @@ -74,13 +87,6 @@ bool AssetManager::initialize(const std::string& dataPath_) { } else { fileCacheBudget = std::clamp(dynamicBudget, minBudgetBytes, maxBudgetBytes); } - - initialized = true; - LOG_INFO("Asset manager initialized (file cache: ", - fileCacheBudget / (1024 * 1024), " MB, recommended=", - recommendedBudget / (1024 * 1024), " MB, max=", - maxBudgetBytes / (1024 * 1024), " MB)"); - return true; } void AssetManager::shutdown() { @@ -90,7 +96,6 @@ void AssetManager::shutdown() { LOG_INFO("Shutting down asset manager"); - // Log cache statistics if (fileCacheHits + fileCacheMisses > 0) { float hitRate = (float)fileCacheHits / (fileCacheHits + fileCacheMisses) * 100.0f; LOG_INFO("File cache stats: ", fileCacheHits, " hits, ", fileCacheMisses, " misses (", @@ -98,8 +103,6 @@ void AssetManager::shutdown() { } clearCache(); - mpqManager.shutdown(); - initialized = false; } @@ -109,19 +112,22 @@ BLPImage AssetManager::loadTexture(const std::string& path) { return BLPImage(); } - // Normalize path std::string normalizedPath = normalizePath(path); LOG_DEBUG("Loading texture: ", normalizedPath); - // Route through readFile() so decompressed MPQ bytes participate in file cache. + // Check for PNG override + BLPImage pngImage = tryLoadPngOverride(normalizedPath); + if (pngImage.isValid()) { + return pngImage; + } + std::vector blpData = readFile(normalizedPath); if (blpData.empty()) { LOG_WARNING("Texture not found: ", normalizedPath); return BLPImage(); } - // Load BLP BLPImage image = BLPLoader::load(blpData); if (!image.isValid()) { LOG_ERROR("Failed to load texture: ", normalizedPath); @@ -132,13 +138,47 @@ BLPImage AssetManager::loadTexture(const std::string& path) { return image; } +BLPImage AssetManager::tryLoadPngOverride(const std::string& normalizedPath) const { + if (normalizedPath.size() < 4) return BLPImage(); + + std::string ext = normalizedPath.substr(normalizedPath.size() - 4); + if (ext != ".blp") return BLPImage(); + + std::string fsPath = manifest_.resolveFilesystemPath(normalizedPath); + if (fsPath.empty()) return BLPImage(); + + // Replace .blp/.BLP extension with .png + std::string pngPath = fsPath.substr(0, fsPath.size() - 4) + ".png"; + if (!LooseFileReader::fileExists(pngPath)) { + return BLPImage(); + } + + int w, h, channels; + unsigned char* pixels = stbi_load(pngPath.c_str(), &w, &h, &channels, 4); + if (!pixels) { + LOG_WARNING("PNG override exists but failed to load: ", pngPath); + return BLPImage(); + } + + BLPImage image; + image.width = w; + image.height = h; + image.channels = 4; + image.format = BLPFormat::BLP2; + image.compression = BLPCompression::ARGB8888; + image.data.assign(pixels, pixels + (w * h * 4)); + stbi_image_free(pixels); + + LOG_INFO("PNG override loaded: ", pngPath, " (", w, "x", h, ")"); + return image; +} + std::shared_ptr AssetManager::loadDBC(const std::string& name) { if (!initialized) { LOG_ERROR("AssetManager not initialized"); return nullptr; } - // Check cache first auto it = dbcCache.find(name); if (it != dbcCache.end()) { LOG_DEBUG("DBC already loaded (cached): ", name); @@ -147,28 +187,20 @@ std::shared_ptr AssetManager::loadDBC(const std::string& name) { LOG_DEBUG("Loading DBC: ", name); - // Construct DBC path (DBFilesClient directory) std::string dbcPath = "DBFilesClient\\" + name; - // Read DBC file from MPQ (must hold readMutex — StormLib is not thread-safe) - std::vector dbcData; - { - std::lock_guard lock(readMutex); - dbcData = mpqManager.readFile(dbcPath); - } + std::vector dbcData = readFile(dbcPath); if (dbcData.empty()) { LOG_WARNING("DBC not found: ", dbcPath); return nullptr; } - // Load DBC auto dbc = std::make_shared(); if (!dbc->load(dbcData)) { LOG_ERROR("Failed to load DBC: ", dbcPath); return nullptr; } - // Cache the DBC dbcCache[name] = dbc; LOG_INFO("Loaded DBC: ", name, " (", dbc->getRecordCount(), " records)"); @@ -187,57 +219,45 @@ bool AssetManager::fileExists(const std::string& path) const { if (!initialized) { return false; } - - std::lock_guard lock(readMutex); - return mpqManager.fileExists(normalizePath(path)); + return manifest_.hasEntry(normalizePath(path)); } std::vector AssetManager::readFile(const std::string& path) const { if (!initialized) { - return std::vector(); + return {}; } std::string normalized = normalizePath(path); + + // Check cache first { std::lock_guard cacheLock(cacheMutex); - // Check cache first auto it = fileCache.find(normalized); if (it != fileCache.end()) { - // Cache hit - update access time and return cached data it->second.lastAccessTime = ++fileCacheAccessCounter; fileCacheHits++; return it->second.data; } } - // Cache miss path: serialize MPQ reads. Before reading, re-check cache while holding - // readMutex so only one thread performs decompression per hot path at a time. - std::vector data; - { - std::lock_guard readLock(readMutex); - { - std::lock_guard cacheLock(cacheMutex); - auto it = fileCache.find(normalized); - if (it != fileCache.end()) { - it->second.lastAccessTime = ++fileCacheAccessCounter; - fileCacheHits++; - return it->second.data; - } - fileCacheMisses++; - } - data = mpqManager.readFile(normalized); + // Read from filesystem (fully parallel, no serialization needed) + std::string fsPath = manifest_.resolveFilesystemPath(normalized); + if (fsPath.empty()) { + return {}; } + + auto data = LooseFileReader::readFile(fsPath); if (data.empty()) { - return data; // File not found + LOG_WARNING("Manifest entry exists but file unreadable: ", fsPath); + return data; } // Add to cache if within budget size_t fileSize = data.size(); - if (fileSize > 0 && fileSize < fileCacheBudget / 2) { // Don't cache files > 50% of budget (very aggressive) + if (fileSize > 0 && fileSize < fileCacheBudget / 2) { std::lock_guard cacheLock(cacheMutex); // Evict old entries if needed (LRU) while (fileCacheTotalBytes + fileSize > fileCacheBudget && !fileCache.empty()) { - // Find least recently used entry auto lru = fileCache.begin(); for (auto it = fileCache.begin(); it != fileCache.end(); ++it) { if (it->second.lastAccessTime < lru->second.lastAccessTime) { @@ -248,7 +268,6 @@ std::vector AssetManager::readFile(const std::string& path) const { fileCache.erase(lru); } - // Add new entry CachedFile cached; cached.data = data; cached.lastAccessTime = ++fileCacheAccessCounter; @@ -261,18 +280,16 @@ std::vector AssetManager::readFile(const std::string& path) const { std::vector AssetManager::readFileOptional(const std::string& path) const { if (!initialized) { - return std::vector(); + return {}; } - - // Avoid MPQManager missing-file warnings for expected probe misses. if (!fileExists(path)) { - return std::vector(); + return {}; } return readFile(path); } void AssetManager::clearCache() { - std::scoped_lock lock(readMutex, cacheMutex); + std::lock_guard lock(cacheMutex); dbcCache.clear(); fileCache.clear(); fileCacheTotalBytes = 0; @@ -282,13 +299,9 @@ void AssetManager::clearCache() { std::string AssetManager::normalizePath(const std::string& path) const { std::string normalized = path; - - // Convert forward slashes to backslashes (WoW uses backslashes) std::replace(normalized.begin(), normalized.end(), '/', '\\'); - // Lowercase for case-insensitive cache keys (improves hit rate across mixed-case callers). std::transform(normalized.begin(), normalized.end(), normalized.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); - return normalized; } diff --git a/src/pipeline/asset_manifest.cpp b/src/pipeline/asset_manifest.cpp new file mode 100644 index 00000000..c6d4bfc6 --- /dev/null +++ b/src/pipeline/asset_manifest.cpp @@ -0,0 +1,97 @@ +#include "pipeline/asset_manifest.hpp" +#include "core/logger.hpp" +#include +#include +#include +#include + +namespace wowee { +namespace pipeline { + +bool AssetManifest::load(const std::string& manifestPath) { + auto startTime = std::chrono::steady_clock::now(); + + std::ifstream file(manifestPath); + if (!file.is_open()) { + LOG_ERROR("Failed to open manifest: ", manifestPath); + return false; + } + + nlohmann::json doc; + try { + doc = nlohmann::json::parse(file); + } catch (const nlohmann::json::parse_error& e) { + LOG_ERROR("Failed to parse manifest JSON: ", e.what()); + return false; + } + + // Read header + int version = doc.value("version", 0); + if (version != 1) { + LOG_ERROR("Unsupported manifest version: ", version); + return false; + } + + basePath_ = doc.value("basePath", "assets"); + manifestDir_ = std::filesystem::path(manifestPath).parent_path().string(); + + // If basePath is relative, resolve against manifest directory + if (!basePath_.empty() && basePath_[0] != '/') { + basePath_ = manifestDir_ + "/" + basePath_; + } + + // Parse entries + auto& entriesObj = doc["entries"]; + if (!entriesObj.is_object()) { + LOG_ERROR("Manifest missing 'entries' object"); + return false; + } + + entries_.reserve(entriesObj.size()); + for (auto& [key, val] : entriesObj.items()) { + Entry entry; + entry.filesystemPath = val.value("p", ""); + entry.size = val.value("s", uint64_t(0)); + + // Parse CRC32 from hex string + std::string hexHash = val.value("h", ""); + if (!hexHash.empty()) { + entry.crc32 = static_cast(std::strtoul(hexHash.c_str(), nullptr, 16)); + } else { + entry.crc32 = 0; + } + + entries_[key] = std::move(entry); + } + + loaded_ = true; + + auto elapsed = std::chrono::steady_clock::now() - startTime; + auto ms = std::chrono::duration_cast(elapsed).count(); + LOG_INFO("Loaded asset manifest: ", entries_.size(), " entries in ", ms, "ms (base: ", basePath_, ")"); + + return true; +} + +const AssetManifest::Entry* AssetManifest::lookup(const std::string& normalizedWowPath) const { + auto it = entries_.find(normalizedWowPath); + if (it != entries_.end()) { + return &it->second; + } + return nullptr; +} + +std::string AssetManifest::resolveFilesystemPath(const std::string& normalizedWowPath) const { + auto it = entries_.find(normalizedWowPath); + if (it == entries_.end()) { + return {}; + } + return basePath_ + "/" + it->second.filesystemPath; +} + +bool AssetManifest::hasEntry(const std::string& normalizedWowPath) const { + return entries_.find(normalizedWowPath) != entries_.end(); +} + +} // namespace pipeline +} // namespace wowee diff --git a/src/pipeline/loose_file_reader.cpp b/src/pipeline/loose_file_reader.cpp new file mode 100644 index 00000000..ded94e7c --- /dev/null +++ b/src/pipeline/loose_file_reader.cpp @@ -0,0 +1,47 @@ +#include "pipeline/loose_file_reader.hpp" +#include "core/logger.hpp" +#include +#include + +namespace wowee { +namespace pipeline { + +std::vector LooseFileReader::readFile(const std::string& filesystemPath) { + std::ifstream file(filesystemPath, std::ios::binary | std::ios::ate); + if (!file.is_open()) { + return {}; + } + + auto size = file.tellg(); + if (size <= 0) { + return {}; + } + + std::vector data(static_cast(size)); + file.seekg(0, std::ios::beg); + file.read(reinterpret_cast(data.data()), size); + + if (!file.good()) { + LOG_WARNING("Incomplete read of: ", filesystemPath); + data.resize(static_cast(file.gcount())); + } + + return data; +} + +bool LooseFileReader::fileExists(const std::string& filesystemPath) { + std::error_code ec; + return std::filesystem::exists(filesystemPath, ec); +} + +uint64_t LooseFileReader::getFileSize(const std::string& filesystemPath) { + std::error_code ec; + auto size = std::filesystem::file_size(filesystemPath, ec); + if (ec) { + return 0; + } + return static_cast(size); +} + +} // namespace pipeline +} // namespace wowee diff --git a/src/rendering/minimap.cpp b/src/rendering/minimap.cpp index b7c47332..96572782 100644 --- a/src/rendering/minimap.cpp +++ b/src/rendering/minimap.cpp @@ -267,7 +267,7 @@ void Minimap::parseTRS() { if (trsParsed || !assetManager) return; trsParsed = true; - auto data = assetManager->getMPQManager().readFile("Textures\\Minimap\\md5translate.trs"); + auto data = assetManager->readFile("Textures\\Minimap\\md5translate.trs"); if (data.empty()) { LOG_WARNING("Failed to load md5translate.trs"); return; diff --git a/tools/asset_extract/extractor.cpp b/tools/asset_extract/extractor.cpp new file mode 100644 index 00000000..73aa5cdc --- /dev/null +++ b/tools/asset_extract/extractor.cpp @@ -0,0 +1,380 @@ +#include "extractor.hpp" +#include "path_mapper.hpp" +#include "manifest_writer.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef INVALID_HANDLE_VALUE +#define INVALID_HANDLE_VALUE ((HANDLE)(long long)-1) +#endif + +namespace wowee { +namespace tools { + +namespace fs = std::filesystem; + +// Archive descriptor for priority-based loading +struct ArchiveDesc { + std::string path; + int priority; +}; + +static std::string toLowerStr(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return s; +} + +static std::string normalizeWowPath(const std::string& path) { + std::string n = path; + std::replace(n.begin(), n.end(), '/', '\\'); + std::transform(n.begin(), n.end(), n.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return n; +} + +// Discover archive files in the same priority order as MPQManager +static std::vector discoverArchives(const std::string& mpqDir) { + std::vector result; + + auto tryAdd = [&](const std::string& name, int prio) { + std::string fullPath = mpqDir + "/" + name; + if (fs::exists(fullPath)) { + result.push_back({fullPath, prio}); + } + }; + + // Base archives (priority 100) + tryAdd("common.MPQ", 100); + tryAdd("common-2.MPQ", 100); + tryAdd("expansion.MPQ", 100); + tryAdd("lichking.MPQ", 100); + + // Patch archives (priority 150-500) + tryAdd("patch.MPQ", 150); + tryAdd("patch-2.MPQ", 200); + tryAdd("patch-3.MPQ", 300); + tryAdd("patch-4.MPQ", 400); + tryAdd("patch-5.MPQ", 500); + + // Letter patches (priority 800-925) + for (char c = 'a'; c <= 'z'; ++c) { + std::string lower = std::string("patch-") + c + ".mpq"; + std::string upper = std::string("Patch-") + static_cast(std::toupper(c)) + ".mpq"; + int prioLower = 800 + (c - 'a'); + int prioUpper = 900 + (c - 'a'); + tryAdd(lower, prioLower); + tryAdd(upper, prioUpper); + } + + // Locale archives + tryAdd("enUS/backup-enUS.MPQ", 230); + tryAdd("enUS/base-enUS.MPQ", 235); + tryAdd("enUS/speech-enUS.MPQ", 240); + tryAdd("enUS/expansion-speech-enUS.MPQ", 245); + tryAdd("enUS/expansion-locale-enUS.MPQ", 246); + tryAdd("enUS/lichking-speech-enUS.MPQ", 248); + tryAdd("enUS/lichking-locale-enUS.MPQ", 249); + tryAdd("enUS/locale-enUS.MPQ", 250); + tryAdd("enUS/patch-enUS.MPQ", 450); + tryAdd("enUS/patch-enUS-2.MPQ", 460); + tryAdd("enUS/patch-enUS-3.MPQ", 470); + + // Sort by priority so highest-priority archives are last + // (we'll iterate highest-prio first when extracting) + std::sort(result.begin(), result.end(), + [](const ArchiveDesc& a, const ArchiveDesc& b) { return a.priority < b.priority; }); + + return result; +} + +bool Extractor::enumerateFiles(const Options& opts, + std::vector& outFiles) { + // Open all archives, enumerate files from highest priority to lowest. + // Use a set to deduplicate (highest-priority version wins). + auto archives = discoverArchives(opts.mpqDir); + if (archives.empty()) { + std::cerr << "No MPQ archives found in: " << opts.mpqDir << "\n"; + return false; + } + + std::cout << "Found " << archives.size() << " MPQ archives\n"; + + // Enumerate from highest priority first so first-seen files win + std::set seenNormalized; + std::vector> fileList; // (original name, archive path) + + for (auto it = archives.rbegin(); it != archives.rend(); ++it) { + HANDLE hMpq = nullptr; + if (!SFileOpenArchive(it->path.c_str(), 0, 0, &hMpq)) { + std::cerr << " Failed to open: " << it->path << "\n"; + continue; + } + + if (opts.verbose) { + std::cout << " Scanning: " << it->path << " (priority " << it->priority << ")\n"; + } + + SFILE_FIND_DATA findData; + HANDLE hFind = SFileFindFirstFile(hMpq, "*", &findData, nullptr); + if (hFind) { + do { + std::string fileName = findData.cFileName; + // Skip internal listfile/attributes + if (fileName == "(listfile)" || fileName == "(attributes)" || + fileName == "(signature)" || fileName == "(patch_metadata)") { + continue; + } + + std::string norm = normalizeWowPath(fileName); + if (seenNormalized.insert(norm).second) { + // First time seeing this file — this is the highest-priority version + outFiles.push_back(fileName); + } + } while (SFileFindNextFile(hFind, &findData)); + SFileFindClose(hFind); + } + + SFileCloseArchive(hMpq); + } + + std::cout << "Enumerated " << outFiles.size() << " unique files\n"; + return true; +} + +bool Extractor::run(const Options& opts) { + auto startTime = std::chrono::steady_clock::now(); + + // Enumerate all unique files across all archives + std::vector files; + if (!enumerateFiles(opts, files)) { + return false; + } + + if (files.empty()) { + std::cerr << "No files to extract\n"; + return false; + } + + // Create output directory + std::error_code ec; + fs::create_directories(opts.outputDir, ec); + if (ec) { + std::cerr << "Failed to create output directory: " << ec.message() << "\n"; + return false; + } + + auto archives = discoverArchives(opts.mpqDir); + + // Determine thread count + int numThreads = opts.threads; + if (numThreads <= 0) { + numThreads = static_cast(std::thread::hardware_concurrency()); + if (numThreads <= 0) numThreads = 4; + } + + Stats stats; + std::mutex manifestMutex; + std::vector manifestEntries; + + // Partition files across threads + std::atomic fileIndex{0}; + size_t totalFiles = files.size(); + + auto workerFn = [&]() { + // Each thread opens ALL archives independently (StormLib is not thread-safe per handle). + // Sorted highest-priority last, so we iterate in reverse to find the winning version. + struct ThreadArchive { + HANDLE handle; + int priority; + }; + std::vector threadHandles; + for (const auto& ad : archives) { + HANDLE h = nullptr; + if (SFileOpenArchive(ad.path.c_str(), 0, 0, &h)) { + threadHandles.push_back({h, ad.priority}); + } + } + if (threadHandles.empty()) { + std::cerr << "Worker thread: failed to open any archives\n"; + return; + } + + while (true) { + size_t idx = fileIndex.fetch_add(1); + if (idx >= totalFiles) break; + + const std::string& wowPath = files[idx]; + std::string normalized = normalizeWowPath(wowPath); + + // Map to new filesystem path + std::string mappedPath = PathMapper::mapPath(wowPath); + std::string fullOutputPath = opts.outputDir + "/" + mappedPath; + + // Search archives in reverse priority order (highest priority first) + HANDLE hFile = nullptr; + for (auto it = threadHandles.rbegin(); it != threadHandles.rend(); ++it) { + if (SFileOpenFileEx(it->handle, wowPath.c_str(), 0, &hFile)) { + break; + } + hFile = nullptr; + } + if (!hFile) { + stats.filesFailed++; + continue; + } + + DWORD fileSize = SFileGetFileSize(hFile, nullptr); + if (fileSize == SFILE_INVALID_SIZE || fileSize == 0) { + SFileCloseFile(hFile); + stats.filesSkipped++; + continue; + } + + std::vector data(fileSize); + DWORD bytesRead = 0; + if (!SFileReadFile(hFile, data.data(), fileSize, &bytesRead, nullptr)) { + SFileCloseFile(hFile); + stats.filesFailed++; + continue; + } + SFileCloseFile(hFile); + data.resize(bytesRead); + + // Create output directory + fs::path outPath(fullOutputPath); + fs::create_directories(outPath.parent_path(), ec); + + // Write file + std::ofstream out(fullOutputPath, std::ios::binary); + if (!out.is_open()) { + stats.filesFailed++; + continue; + } + out.write(reinterpret_cast(data.data()), data.size()); + out.close(); + + // Compute CRC32 + uint32_t crc = ManifestWriter::computeCRC32(data.data(), data.size()); + + // Add manifest entry + ManifestWriter::FileEntry entry; + entry.wowPath = normalized; + entry.filesystemPath = mappedPath; + entry.size = data.size(); + entry.crc32 = crc; + + { + std::lock_guard lock(manifestMutex); + manifestEntries.push_back(std::move(entry)); + } + + stats.filesExtracted++; + stats.bytesExtracted += data.size(); + + // Progress + uint64_t done = stats.filesExtracted.load(); + if (done % 1000 == 0) { + std::cout << "\r Extracted " << done << " / " << totalFiles << " files..." + << std::flush; + } + } + + for (auto& th : threadHandles) { + SFileCloseArchive(th.handle); + } + }; + + std::cout << "Extracting " << totalFiles << " files using " << numThreads << " threads...\n"; + + std::vector threads; + for (int i = 0; i < numThreads; ++i) { + threads.emplace_back(workerFn); + } + for (auto& t : threads) { + t.join(); + } + + std::cout << "\r Extracted " << stats.filesExtracted.load() << " files (" + << stats.bytesExtracted.load() / (1024 * 1024) << " MB), " + << stats.filesSkipped.load() << " skipped, " + << stats.filesFailed.load() << " failed\n"; + + // Sort manifest entries for deterministic output + std::sort(manifestEntries.begin(), manifestEntries.end(), + [](const ManifestWriter::FileEntry& a, const ManifestWriter::FileEntry& b) { + return a.wowPath < b.wowPath; + }); + + // Write manifest + std::string manifestPath = opts.outputDir + "/manifest.json"; + // basePath is "." since manifest sits inside the output directory + if (!ManifestWriter::write(manifestPath, ".", manifestEntries)) { + std::cerr << "Failed to write manifest: " << manifestPath << "\n"; + return false; + } + + std::cout << "Wrote manifest: " << manifestPath << " (" << manifestEntries.size() << " entries)\n"; + + // Verification pass + if (opts.verify) { + std::cout << "Verifying extracted files...\n"; + uint64_t verified = 0, verifyFailed = 0; + for (const auto& entry : manifestEntries) { + std::string fsPath = opts.outputDir + "/" + entry.filesystemPath; + std::ifstream f(fsPath, std::ios::binary | std::ios::ate); + if (!f.is_open()) { + std::cerr << " MISSING: " << fsPath << "\n"; + verifyFailed++; + continue; + } + + auto size = f.tellg(); + if (static_cast(size) != entry.size) { + std::cerr << " SIZE MISMATCH: " << fsPath << " (expected " + << entry.size << ", got " << size << ")\n"; + verifyFailed++; + continue; + } + + f.seekg(0); + std::vector data(static_cast(size)); + f.read(reinterpret_cast(data.data()), size); + + uint32_t crc = ManifestWriter::computeCRC32(data.data(), data.size()); + if (crc != entry.crc32) { + std::cerr << " CRC MISMATCH: " << fsPath << "\n"; + verifyFailed++; + continue; + } + + verified++; + } + std::cout << "Verified " << verified << " files"; + if (verifyFailed > 0) { + std::cout << " (" << verifyFailed << " FAILED)"; + } + std::cout << "\n"; + } + + auto elapsed = std::chrono::steady_clock::now() - startTime; + auto secs = std::chrono::duration_cast(elapsed).count(); + std::cout << "Done in " << secs / 60 << "m " << secs % 60 << "s\n"; + + return true; +} + +} // namespace tools +} // namespace wowee diff --git a/tools/asset_extract/extractor.hpp b/tools/asset_extract/extractor.hpp new file mode 100644 index 00000000..a44ef87d --- /dev/null +++ b/tools/asset_extract/extractor.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include + +namespace wowee { +namespace tools { + +/** + * Extraction pipeline: MPQ archives → loose files + manifest + */ +class Extractor { +public: + struct Options { + std::string mpqDir; // Path to WoW Data directory + std::string outputDir; // Output directory for extracted assets + int threads = 0; // 0 = auto-detect + bool verify = false; // CRC32 verify after extraction + bool verbose = false; // Verbose logging + }; + + struct Stats { + std::atomic filesExtracted{0}; + std::atomic bytesExtracted{0}; + std::atomic filesSkipped{0}; + std::atomic filesFailed{0}; + }; + + /** + * Run the extraction pipeline + * @return true on success + */ + static bool run(const Options& opts); + +private: + static bool enumerateFiles(const Options& opts, + std::vector& outFiles); +}; + +} // namespace tools +} // namespace wowee diff --git a/tools/asset_extract/main.cpp b/tools/asset_extract/main.cpp new file mode 100644 index 00000000..c804139a --- /dev/null +++ b/tools/asset_extract/main.cpp @@ -0,0 +1,62 @@ +#include "extractor.hpp" +#include +#include +#include + +static void printUsage(const char* prog) { + std::cout << "Usage: " << prog << " --mpq-dir --output [options]\n" + << "\n" + << "Extract WoW MPQ archives to organized loose files with manifest.\n" + << "\n" + << "Required:\n" + << " --mpq-dir Path to WoW Data directory containing MPQ files\n" + << " --output Output directory for extracted assets\n" + << "\n" + << "Options:\n" + << " --verify CRC32 verify all extracted files\n" + << " --threads Number of extraction threads (default: auto)\n" + << " --verbose Verbose output\n" + << " --help Show this help\n"; +} + +int main(int argc, char** argv) { + wowee::tools::Extractor::Options opts; + + for (int i = 1; i < argc; ++i) { + if (std::strcmp(argv[i], "--mpq-dir") == 0 && i + 1 < argc) { + opts.mpqDir = argv[++i]; + } else if (std::strcmp(argv[i], "--output") == 0 && i + 1 < argc) { + opts.outputDir = argv[++i]; + } else if (std::strcmp(argv[i], "--threads") == 0 && i + 1 < argc) { + opts.threads = std::atoi(argv[++i]); + } else if (std::strcmp(argv[i], "--verify") == 0) { + opts.verify = true; + } else if (std::strcmp(argv[i], "--verbose") == 0) { + opts.verbose = true; + } else if (std::strcmp(argv[i], "--help") == 0 || std::strcmp(argv[i], "-h") == 0) { + printUsage(argv[0]); + return 0; + } else { + std::cerr << "Unknown option: " << argv[i] << "\n"; + printUsage(argv[0]); + return 1; + } + } + + if (opts.mpqDir.empty() || opts.outputDir.empty()) { + std::cerr << "Error: --mpq-dir and --output are required\n\n"; + printUsage(argv[0]); + return 1; + } + + std::cout << "=== Wowee Asset Extractor ===\n"; + std::cout << "MPQ directory: " << opts.mpqDir << "\n"; + std::cout << "Output: " << opts.outputDir << "\n"; + + if (!wowee::tools::Extractor::run(opts)) { + std::cerr << "Extraction failed!\n"; + return 1; + } + + return 0; +} diff --git a/tools/asset_extract/manifest_writer.cpp b/tools/asset_extract/manifest_writer.cpp new file mode 100644 index 00000000..e5b1fca5 --- /dev/null +++ b/tools/asset_extract/manifest_writer.cpp @@ -0,0 +1,67 @@ +#include "manifest_writer.hpp" +#include +#include +#include +#include + +namespace wowee { +namespace tools { + +uint32_t ManifestWriter::computeCRC32(const uint8_t* data, size_t size) { + return static_cast(::crc32(::crc32(0L, Z_NULL, 0), data, static_cast(size))); +} + +bool ManifestWriter::write(const std::string& outputPath, + const std::string& basePath, + const std::vector& entries) { + // Write JSON manually to avoid pulling nlohmann/json into the tool + // (though it would also work fine). This keeps the tool dependency-light. + std::ofstream file(outputPath); + if (!file.is_open()) { + return false; + } + + file << "{\n"; + file << " \"version\": 1,\n"; + file << " \"basePath\": \"" << basePath << "\",\n"; + file << " \"fileCount\": " << entries.size() << ",\n"; + file << " \"entries\": {\n"; + + for (size_t i = 0; i < entries.size(); ++i) { + const auto& e = entries[i]; + + // Escape backslashes in WoW path for JSON + std::string escapedKey; + for (char c : e.wowPath) { + if (c == '\\') escapedKey += "\\\\"; + else if (c == '"') escapedKey += "\\\""; + else escapedKey += c; + } + + std::string escapedPath; + for (char c : e.filesystemPath) { + if (c == '\\') escapedPath += "\\\\"; + else if (c == '"') escapedPath += "\\\""; + else escapedPath += c; + } + + // CRC32 as hex + std::ostringstream hexCrc; + hexCrc << std::hex << std::setfill('0') << std::setw(8) << e.crc32; + + file << " \"" << escapedKey << "\": {\"p\": \"" << escapedPath + << "\", \"s\": " << e.size + << ", \"h\": \"" << hexCrc.str() << "\"}"; + + if (i + 1 < entries.size()) file << ","; + file << "\n"; + } + + file << " }\n"; + file << "}\n"; + + return file.good(); +} + +} // namespace tools +} // namespace wowee diff --git a/tools/asset_extract/manifest_writer.hpp b/tools/asset_extract/manifest_writer.hpp new file mode 100644 index 00000000..8de52835 --- /dev/null +++ b/tools/asset_extract/manifest_writer.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +namespace wowee { +namespace tools { + +/** + * Generates manifest.json from extracted file metadata. + */ +class ManifestWriter { +public: + struct FileEntry { + std::string wowPath; // Normalized WoW virtual path (lowercase, backslash) + std::string filesystemPath; // Relative path from basePath (forward slashes, original case) + uint64_t size; // File size in bytes + uint32_t crc32; // CRC32 checksum + }; + + /** + * Write manifest.json + * @param outputPath Full path to manifest.json + * @param basePath Value for basePath field (e.g., "assets") + * @param entries All extracted file entries + * @return true on success + */ + static bool write(const std::string& outputPath, + const std::string& basePath, + const std::vector& entries); + + /** + * Compute CRC32 of file data + */ + static uint32_t computeCRC32(const uint8_t* data, size_t size); +}; + +} // namespace tools +} // namespace wowee diff --git a/tools/asset_extract/path_mapper.cpp b/tools/asset_extract/path_mapper.cpp new file mode 100644 index 00000000..e4e1c8c4 --- /dev/null +++ b/tools/asset_extract/path_mapper.cpp @@ -0,0 +1,207 @@ +#include "path_mapper.hpp" +#include +#include + +namespace wowee { +namespace tools { + +std::string PathMapper::toLower(const std::string& str) { + std::string result = str; + std::transform(result.begin(), result.end(), result.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return result; +} + +std::string PathMapper::toForwardSlash(const std::string& str) { + std::string result = str; + std::replace(result.begin(), result.end(), '\\', '/'); + return result; +} + +bool PathMapper::startsWithCI(const std::string& str, const std::string& prefix) { + if (str.size() < prefix.size()) return false; + for (size_t i = 0; i < prefix.size(); ++i) { + if (std::tolower(static_cast(str[i])) != + std::tolower(static_cast(prefix[i]))) { + return false; + } + } + return true; +} + +std::string PathMapper::extractAfterPrefix(const std::string& path, size_t prefixLen) { + if (prefixLen >= path.size()) return {}; + return path.substr(prefixLen); +} + +std::string PathMapper::mapPath(const std::string& wowPath) { + // Preserve original casing in the remainder for filesystem readability + std::string rest; + + // DBFilesClient\ → db/ + if (startsWithCI(wowPath, "DBFilesClient\\")) { + rest = extractAfterPrefix(wowPath, 14); + return "db/" + toForwardSlash(rest); + } + + // Character\{Race}\{Gender}\ → character/{race}/{gender}/ + if (startsWithCI(wowPath, "Character\\")) { + rest = extractAfterPrefix(wowPath, 10); + std::string lowered = toLower(rest); + return "character/" + toForwardSlash(lowered); + } + + // Creature\{Name}\ → creature/{name}/ + if (startsWithCI(wowPath, "Creature\\")) { + rest = extractAfterPrefix(wowPath, 9); + // Keep first component lowercase for directory, preserve filename case + std::string fwd = toForwardSlash(rest); + auto slash = fwd.find('/'); + if (slash != std::string::npos) { + return "creature/" + toLower(fwd.substr(0, slash)) + "/" + fwd.substr(slash + 1); + } + return "creature/" + fwd; + } + + // Item\ObjectComponents\{Type}\ → item/{type}/ + if (startsWithCI(wowPath, "Item\\ObjectComponents\\")) { + rest = extractAfterPrefix(wowPath, 22); + std::string fwd = toForwardSlash(rest); + auto slash = fwd.find('/'); + if (slash != std::string::npos) { + return "item/" + toLower(fwd.substr(0, slash)) + "/" + fwd.substr(slash + 1); + } + return "item/" + fwd; + } + + // Item\TextureComponents\ → item/texture/ + if (startsWithCI(wowPath, "Item\\TextureComponents\\")) { + rest = extractAfterPrefix(wowPath, 23); + return "item/texture/" + toForwardSlash(rest); + } + + // Interface\Icons\ → interface/icons/ + if (startsWithCI(wowPath, "Interface\\Icons\\")) { + rest = extractAfterPrefix(wowPath, 16); + return "interface/icons/" + toForwardSlash(rest); + } + + // Interface\GossipFrame\ → interface/gossip/ + if (startsWithCI(wowPath, "Interface\\GossipFrame\\")) { + rest = extractAfterPrefix(wowPath, 21); + return "interface/gossip/" + toForwardSlash(rest); + } + + // Interface\{rest} → interface/{rest}/ + if (startsWithCI(wowPath, "Interface\\")) { + rest = extractAfterPrefix(wowPath, 10); + return "interface/" + toForwardSlash(rest); + } + + // Textures\Minimap\ → terrain/minimap/ + if (startsWithCI(wowPath, "Textures\\Minimap\\")) { + rest = extractAfterPrefix(wowPath, 17); + return "terrain/minimap/" + toForwardSlash(rest); + } + + // Textures\BakedNpcTextures\ → creature/baked/ + if (startsWithCI(wowPath, "Textures\\BakedNpcTextures\\")) { + rest = extractAfterPrefix(wowPath, 25); + return "creature/baked/" + toForwardSlash(rest); + } + + // Textures\{rest} → terrain/textures/{rest} + if (startsWithCI(wowPath, "Textures\\")) { + rest = extractAfterPrefix(wowPath, 9); + return "terrain/textures/" + toForwardSlash(rest); + } + + // World\Maps\{Map}\ → terrain/maps/{map}/ + if (startsWithCI(wowPath, "World\\Maps\\")) { + rest = extractAfterPrefix(wowPath, 11); + std::string fwd = toForwardSlash(rest); + auto slash = fwd.find('/'); + if (slash != std::string::npos) { + return "terrain/maps/" + toLower(fwd.substr(0, slash)) + "/" + fwd.substr(slash + 1); + } + return "terrain/maps/" + fwd; + } + + // World\wmo\ → world/wmo/ (preserve subpath) + if (startsWithCI(wowPath, "World\\wmo\\")) { + rest = extractAfterPrefix(wowPath, 10); + return "world/wmo/" + toForwardSlash(rest); + } + + // World\Doodads\ → world/doodads/ + if (startsWithCI(wowPath, "World\\Doodads\\")) { + rest = extractAfterPrefix(wowPath, 14); + return "world/doodads/" + toForwardSlash(rest); + } + + // World\{rest} → world/{rest}/ + if (startsWithCI(wowPath, "World\\")) { + rest = extractAfterPrefix(wowPath, 6); + return "world/" + toForwardSlash(rest); + } + + // Environments\ → environment/ + if (startsWithCI(wowPath, "Environments\\")) { + rest = extractAfterPrefix(wowPath, 13); + return "environment/" + toForwardSlash(rest); + } + + // Sound\Ambience\ → sound/ambient/ + if (startsWithCI(wowPath, "Sound\\Ambience\\")) { + rest = extractAfterPrefix(wowPath, 15); + return "sound/ambient/" + toForwardSlash(rest); + } + + // Sound\Character\ → sound/character/ + if (startsWithCI(wowPath, "Sound\\Character\\")) { + rest = extractAfterPrefix(wowPath, 16); + return "sound/character/" + toForwardSlash(rest); + } + + // Sound\Doodad\ → sound/doodad/ + if (startsWithCI(wowPath, "Sound\\Doodad\\")) { + rest = extractAfterPrefix(wowPath, 13); + return "sound/doodad/" + toForwardSlash(rest); + } + + // Sound\Creature\ → sound/creature/ + if (startsWithCI(wowPath, "Sound\\Creature\\")) { + rest = extractAfterPrefix(wowPath, 15); + return "sound/creature/" + toForwardSlash(rest); + } + + // Sound\Spells\ → sound/spell/ + if (startsWithCI(wowPath, "Sound\\Spells\\")) { + rest = extractAfterPrefix(wowPath, 13); + return "sound/spell/" + toForwardSlash(rest); + } + + // Sound\Music\ → sound/music/ + if (startsWithCI(wowPath, "Sound\\Music\\")) { + rest = extractAfterPrefix(wowPath, 12); + return "sound/music/" + toForwardSlash(rest); + } + + // Sound\{rest} → sound/{rest}/ + if (startsWithCI(wowPath, "Sound\\")) { + rest = extractAfterPrefix(wowPath, 6); + return "sound/" + toForwardSlash(rest); + } + + // Spells\ → spell/ + if (startsWithCI(wowPath, "Spells\\")) { + rest = extractAfterPrefix(wowPath, 7); + return "spell/" + toForwardSlash(rest); + } + + // Everything else → misc/{original_path} + return "misc/" + toForwardSlash(wowPath); +} + +} // namespace tools +} // namespace wowee diff --git a/tools/asset_extract/path_mapper.hpp b/tools/asset_extract/path_mapper.hpp new file mode 100644 index 00000000..cad31c57 --- /dev/null +++ b/tools/asset_extract/path_mapper.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace wowee { +namespace tools { + +/** + * Maps WoW virtual paths to reorganized filesystem categories. + * + * Input: WoW virtual path (e.g., "Creature\\Bear\\BearSkin.blp") + * Output: Category-based relative path (e.g., "creature/bear/BearSkin.blp") + */ +class PathMapper { +public: + /** + * Map a WoW virtual path to a reorganized filesystem path. + * @param wowPath Original WoW virtual path (backslash-separated) + * @return Reorganized relative path (forward-slash separated, original casing preserved) + */ + static std::string mapPath(const std::string& wowPath); + +private: + // Helpers for prefix matching (case-insensitive) + static bool startsWithCI(const std::string& str, const std::string& prefix); + static std::string toLower(const std::string& str); + static std::string toForwardSlash(const std::string& str); + static std::string extractAfterPrefix(const std::string& path, size_t prefixLen); +}; + +} // namespace tools +} // namespace wowee diff --git a/tools/blp_convert/main.cpp b/tools/blp_convert/main.cpp new file mode 100644 index 00000000..a82d457a --- /dev/null +++ b/tools/blp_convert/main.cpp @@ -0,0 +1,187 @@ +#include "pipeline/blp_loader.hpp" + +#define STB_IMAGE_IMPLEMENTATION +#include "stb_image.h" +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include "stb_image_write.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +using wowee::pipeline::BLPImage; +using wowee::pipeline::BLPLoader; + +static std::vector readFileData(const std::string& path) { + std::ifstream f(path, std::ios::binary | std::ios::ate); + if (!f.is_open()) return {}; + auto sz = f.tellg(); + if (sz <= 0) return {}; + std::vector data(static_cast(sz)); + f.seekg(0); + f.read(reinterpret_cast(data.data()), sz); + return data; +} + +static bool convertBlpToPng(const std::string& blpPath) { + auto data = readFileData(blpPath); + if (data.empty()) { + std::cerr << "Failed to read: " << blpPath << "\n"; + return false; + } + + BLPImage img = BLPLoader::load(data); + if (!img.isValid()) { + std::cerr << "Failed to decode BLP: " << blpPath << "\n"; + return false; + } + + // Output path: same name with .png + fs::path out = fs::path(blpPath); + out.replace_extension(".png"); + + if (!stbi_write_png(out.string().c_str(), img.width, img.height, 4, + img.data.data(), img.width * 4)) { + std::cerr << "Failed to write PNG: " << out << "\n"; + return false; + } + + std::cout << blpPath << " -> " << out.string() << " (" << img.width << "x" << img.height << ")\n"; + return true; +} + +static bool convertPngToBlp(const std::string& pngPath) { + // Load PNG + int w, h, channels; + unsigned char* pixels = stbi_load(pngPath.c_str(), &w, &h, &channels, 4); + if (!pixels) { + std::cerr << "Failed to read PNG: " << pngPath << "\n"; + return false; + } + + // Write a simple uncompressed BLP2 (ARGB8888) + fs::path out = fs::path(pngPath); + out.replace_extension(".blp"); + + std::ofstream f(out.string(), std::ios::binary); + if (!f.is_open()) { + stbi_image_free(pixels); + std::cerr << "Failed to open output: " << out << "\n"; + return false; + } + + // BLP2 header + f.write("BLP2", 4); + uint32_t version = 1; + f.write(reinterpret_cast(&version), 4); + + uint8_t compression = 3; // uncompressed + uint8_t alphaDepth = 8; + uint8_t alphaEncoding = 0; + uint8_t hasMips = 0; + f.write(reinterpret_cast(&compression), 1); + f.write(reinterpret_cast(&alphaDepth), 1); + f.write(reinterpret_cast(&alphaEncoding), 1); + f.write(reinterpret_cast(&hasMips), 1); + + uint32_t width = static_cast(w); + uint32_t height = static_cast(h); + f.write(reinterpret_cast(&width), 4); + f.write(reinterpret_cast(&height), 4); + + // Mip offsets (16 entries) — only first used + uint32_t dataSize = width * height * 4; + uint32_t headerSize = 4 + 4 + 4 + 4 + 16 * 4 + 16 * 4 + 256 * 4; // magic+version+dims+mips+palette + uint32_t mipOffsets[16] = {}; + uint32_t mipSizes[16] = {}; + mipOffsets[0] = headerSize; + mipSizes[0] = dataSize; + f.write(reinterpret_cast(mipOffsets), sizeof(mipOffsets)); + f.write(reinterpret_cast(mipSizes), sizeof(mipSizes)); + + // Empty palette (256 entries) + uint32_t palette[256] = {}; + f.write(reinterpret_cast(palette), sizeof(palette)); + + // Convert RGBA → BGRA for BLP + std::vector bgra(dataSize); + for (int i = 0; i < w * h; ++i) { + bgra[i * 4 + 0] = pixels[i * 4 + 2]; // B + bgra[i * 4 + 1] = pixels[i * 4 + 1]; // G + bgra[i * 4 + 2] = pixels[i * 4 + 0]; // R + bgra[i * 4 + 3] = pixels[i * 4 + 3]; // A + } + f.write(reinterpret_cast(bgra.data()), dataSize); + + stbi_image_free(pixels); + + std::cout << pngPath << " -> " << out.string() << " (" << w << "x" << h << ")\n"; + return true; +} + +static void printUsage(const char* prog) { + std::cout << "Usage:\n" + << " " << prog << " --to-png Convert BLP to PNG\n" + << " " << prog << " --to-blp Convert PNG to BLP\n" + << " " << prog << " --batch [--recursive] Batch convert BLP->PNG\n"; +} + +int main(int argc, char** argv) { + if (argc < 3) { + printUsage(argv[0]); + return 1; + } + + std::string mode = argv[1]; + + if (mode == "--to-png") { + return convertBlpToPng(argv[2]) ? 0 : 1; + } + + if (mode == "--to-blp") { + return convertPngToBlp(argv[2]) ? 0 : 1; + } + + if (mode == "--batch") { + std::string dir = argv[2]; + bool recursive = (argc > 3 && std::strcmp(argv[3], "--recursive") == 0); + + int count = 0, failed = 0; + auto processEntry = [&](const fs::directory_entry& entry) { + if (!entry.is_regular_file()) return; + std::string ext = entry.path().extension().string(); + std::transform(ext.begin(), ext.end(), ext.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (ext == ".blp") { + if (convertBlpToPng(entry.path().string())) { + count++; + } else { + failed++; + } + } + }; + + if (recursive) { + for (const auto& entry : fs::recursive_directory_iterator(dir)) { + processEntry(entry); + } + } else { + for (const auto& entry : fs::directory_iterator(dir)) { + processEntry(entry); + } + } + + std::cout << "Batch complete: " << count << " converted, " << failed << " failed\n"; + return failed > 0 ? 1 : 0; + } + + std::cerr << "Unknown mode: " << mode << "\n"; + printUsage(argv[0]); + return 1; +}