phaneron/Kelsidavis-WoWee
1
0
Fork 0
mirror of https://github.com/Kelsidavis/WoWee.git synced 2026-03-22 23:30:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Replace MPQ runtime with loose file asset system

Browse source 
Extract assets from MPQ archives into organized loose files indexed by
manifest.json, enabling fully parallel reads without StormLib serialization.
Add asset_extract and blp_convert tools, PNG texture override support.
This commit is contained in:
Kelsi 2026-02-12 20:32:14 -08:00
parent 5fda1a3157
commit aa16a687c2
16 changed files with 1427 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

62
CMakeLists.txt
View file

@ -43,15 +43,10 @@ if(NOT glm_FOUND)
message(STATUS "GLM not found, will use system includes or download")
endif()
# StormLib for MPQ archives
# StormLib for MPQ extraction tool (not needed for main executable)
find_library(STORMLIB_LIBRARY NAMES StormLib stormlib storm)
find_path(STORMLIB_INCLUDE_DIR StormLib.h PATH_SUFFIXES StormLib)
if(NOT STORMLIB_LIBRARY OR NOT STORMLIB_INCLUDE_DIR)
message(WARNING "StormLib not found. You may need to build it manually.")
message(WARNING "Get it from: https://github.com/ladislav-zezula/StormLib")
endif()
# Include ImGui as a static library (we'll add the sources)
set(IMGUI_DIR ${CMAKE_CURRENT_SOURCE_DIR}/extern/imgui)
if(EXISTS ${IMGUI_DIR})
@ -128,10 +123,11 @@ set(WOWEE_SOURCES
src/audio/movement_sound_manager.cpp
# Pipeline (asset loaders)
src/pipeline/mpq_manager.cpp
src/pipeline/blp_loader.cpp
src/pipeline/dbc_loader.cpp
src/pipeline/asset_manager.cpp
src/pipeline/asset_manifest.cpp
src/pipeline/loose_file_reader.cpp
src/pipeline/m2_loader.cpp
src/pipeline/wmo_loader.cpp
src/pipeline/adt_loader.cpp
@ -233,8 +229,9 @@ set(WOWEE_HEADERS
include/audio/spell_sound_manager.hpp
include/audio/movement_sound_manager.hpp
include/pipeline/mpq_manager.hpp
include/pipeline/blp_loader.hpp
include/pipeline/asset_manifest.hpp
include/pipeline/loose_file_reader.hpp
include/pipeline/m2_loader.hpp
include/pipeline/wmo_loader.hpp
include/pipeline/adt_loader.hpp
@ -325,13 +322,6 @@ if(WIN32)
endif()
endif()
# Link StormLib if found
if(STORMLIB_LIBRARY AND STORMLIB_INCLUDE_DIR)
target_link_libraries(wowee PRIVATE ${STORMLIB_LIBRARY})
target_include_directories(wowee PRIVATE ${STORMLIB_INCLUDE_DIR})
target_compile_definitions(wowee PRIVATE HAVE_STORMLIB)
endif()
# Link ImGui if available
if(TARGET imgui)
target_link_libraries(wowee PRIVATE imgui)
@ -385,6 +375,47 @@ if(UNIX AND NOT APPLE)
RENAME wowee.png)
endif()
# ---- Tool: asset_extract (MPQ → loose files) ----
if(STORMLIB_LIBRARY AND STORMLIB_INCLUDE_DIR)
add_executable(asset_extract
tools/asset_extract/main.cpp
tools/asset_extract/extractor.cpp
tools/asset_extract/path_mapper.cpp
tools/asset_extract/manifest_writer.cpp
)
target_include_directories(asset_extract PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/tools/asset_extract
${STORMLIB_INCLUDE_DIR}
)
target_link_libraries(asset_extract PRIVATE
${STORMLIB_LIBRARY}
ZLIB::ZLIB
Threads::Threads
)
set_target_properties(asset_extract PROPERTIES
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin
)
message(STATUS " asset_extract tool: ENABLED")
else()
message(STATUS " asset_extract tool: DISABLED (requires StormLib)")
endif()
# ---- Tool: blp_convert (BLP ↔ PNG) ----
add_executable(blp_convert
tools/blp_convert/main.cpp
src/pipeline/blp_loader.cpp
src/core/logger.cpp
)
target_include_directories(blp_convert PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/extern
)
target_link_libraries(blp_convert PRIVATE Threads::Threads)
set_target_properties(blp_convert PROPERTIES
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin
)
# Print configuration summary
message(STATUS "")
message(STATUS "Wowee Configuration:")
@ -392,6 +423,5 @@ message(STATUS " C++ Standard: ${CMAKE_CXX_STANDARD}")
message(STATUS " Build Type: ${CMAKE_BUILD_TYPE}")
message(STATUS " SDL2: ${SDL2_VERSION}")
message(STATUS " OpenSSL: ${OPENSSL_VERSION}")
message(STATUS " StormLib: ${STORMLIB_LIBRARY}")
message(STATUS " ImGui: ${IMGUI_DIR}")
message(STATUS "")

37
include/pipeline/asset_manager.hpp
View file

@ -1,8 +1,9 @@
#pragma once
#include "pipeline/mpq_manager.hpp"
#include "pipeline/blp_loader.hpp"
#include "pipeline/dbc_loader.hpp"
#include "pipeline/asset_manifest.hpp"
#include "pipeline/loose_file_reader.hpp"
#include <memory>
#include <string>
#include <map>
@ -14,7 +15,9 @@ namespace pipeline {
/**
* AssetManager - Unified interface for loading WoW assets
*
* Coordinates MPQ archives, texture loading, and database files
* Reads pre-extracted loose files indexed by manifest.json.
* Use the asset_extract tool to extract MPQ archives first.
* All reads are fully parallel (no serialization mutex needed).
*/
class AssetManager {
public:
@ -23,7 +26,7 @@ public:
/**
* Initialize asset manager
* @param dataPath Path to WoW Data directory
* @param dataPath Path to directory containing manifest.json and extracted assets
* @return true if initialization succeeded
*/
bool initialize(const std::string& dataPath);
@ -60,33 +63,27 @@ public:
std::shared_ptr<DBCFile> getDBC(const std::string& name) const;
/**
* Check if a file exists in MPQ archives
* Check if a file exists
* @param path Virtual file path
* @return true if file exists
*/
bool fileExists(const std::string& path) const;
/**
* Read raw file data from MPQ archives
* Read raw file data
* @param path Virtual file path
* @return File contents (empty if not found)
*/
std::vector<uint8_t> readFile(const std::string& path) const;
/**
* Read optional file data from MPQ archives without warning spam.
* Read optional file data without warning spam.
* Intended for probe-style lookups (e.g. external .anim variants).
* @param path Virtual file path
* @return File contents (empty if not found)
*/
std::vector<uint8_t> readFileOptional(const std::string& path) const;
/**
* Get MPQ manager for direct access
*/
MPQManager& getMPQManager() { return mpqManager; }
const MPQManager& getMPQManager() const { return mpqManager; }
/**
* Get loaded DBC count
*/
@ -108,12 +105,14 @@ private:
bool initialized = false;
std::string dataPath;
MPQManager mpqManager;
mutable std::mutex readMutex;
// Loose file backend
AssetManifest manifest_;
LooseFileReader looseReader_;
mutable std::mutex cacheMutex;
std::map<std::string, std::shared_ptr<DBCFile>> dbcCache;
// Decompressed file cache (LRU, dynamic budget based on system RAM)
// File cache (LRU, dynamic budget based on system RAM)
struct CachedFile {
std::vector<uint8_t> data;
uint64_t lastAccessTime;
@ -125,6 +124,14 @@ private:
mutable size_t fileCacheMisses = 0;
mutable size_t fileCacheBudget = 1024 * 1024 * 1024; // Dynamic, starts at 1GB
void setupFileCacheBudget();
/**
* Try to load a PNG override for a BLP path.
* Returns valid BLPImage if PNG found, invalid otherwise.
*/
BLPImage tryLoadPngOverride(const std::string& normalizedPath) const;
/**
* Normalize path for case-insensitive lookup
*/

74
include/pipeline/asset_manifest.hpp Normal file
View file

@ -0,0 +1,74 @@
#pragma once
#include <string>
#include <vector>
#include <cstdint>
#include <unordered_map>
namespace wowee {
namespace pipeline {
/**
* AssetManifest - Maps WoW virtual paths to filesystem paths
*
* Loaded once at startup from manifest.json. Read-only after init,
* so concurrent reads are safe without a mutex.
*/
class AssetManifest {
public:
struct Entry {
std::string filesystemPath; // Relative path from basePath (forward slashes)
uint64_t size; // File size in bytes
uint32_t crc32; // CRC32 for integrity verification
};
AssetManifest() = default;
/**
* Load manifest from JSON file
* @param manifestPath Full path to manifest.json
* @return true if loaded successfully
*/
bool load(const std::string& manifestPath);
/**
* Lookup an entry by normalized WoW path (lowercase, backslash)
* @return Pointer to entry or nullptr if not found
*/
const Entry* lookup(const std::string& normalizedWowPath) const;
/**
* Resolve full filesystem path for a WoW virtual path
* @return Full filesystem path or empty string if not found
*/
std::string resolveFilesystemPath(const std::string& normalizedWowPath) const;
/**
* Check if an entry exists
*/
bool hasEntry(const std::string& normalizedWowPath) const;
/**
* Get base path (directory containing extracted assets)
*/
const std::string& getBasePath() const { return basePath_; }
/**
* Get total number of entries
*/
size_t getEntryCount() const { return entries_.size(); }
/**
* Check if manifest is loaded
*/
bool isLoaded() const { return loaded_; }
private:
bool loaded_ = false;
std::string basePath_; // Root directory for extracted assets
std::string manifestDir_; // Directory containing manifest.json
std::unordered_map<std::string, Entry> entries_;
};
} // namespace pipeline
} // namespace wowee

40
include/pipeline/loose_file_reader.hpp Normal file
View file

@ -0,0 +1,40 @@
#pragma once
#include <string>
#include <vector>
#include <cstdint>
namespace wowee {
namespace pipeline {
/**
* LooseFileReader - Thread-safe filesystem file reader
*
* Each read opens its own file descriptor, so no mutex is needed.
* This replaces the serialized MPQ read path.
*/
class LooseFileReader {
public:
LooseFileReader() = default;
/**
* Read entire file into memory
* @param filesystemPath Full path to file on disk
* @return File contents (empty if not found or error)
*/
static std::vector<uint8_t> readFile(const std::string& filesystemPath);
/**
* Check if a file exists on disk
*/
static bool fileExists(const std::string& filesystemPath);
/**
* Get file size without reading
* @return Size in bytes, or 0 if not found
*/
static uint64_t getFileSize(const std::string& filesystemPath);
};
} // namespace pipeline
} // namespace wowee

151
src/pipeline/asset_manager.cpp
View file

@ -3,8 +3,11 @@
#include "core/memory_monitor.hpp"
#include <algorithm>
#include <cstdlib>
#include <filesystem>
#include <limits>
#include "stb_image.h"
namespace wowee {
namespace pipeline {
@ -40,26 +43,36 @@ bool AssetManager::initialize(const std::string& dataPath_) {
dataPath = dataPath_;
LOG_INFO("Initializing asset manager with data path: ", dataPath);
// Initialize MPQ manager
if (!mpqManager.initialize(dataPath)) {
LOG_ERROR("Failed to initialize MPQ manager");
setupFileCacheBudget();
std::string manifestPath = dataPath + "/manifest.json";
if (!std::filesystem::exists(manifestPath)) {
LOG_ERROR("manifest.json not found in: ", dataPath);
LOG_ERROR("Run asset_extract to extract MPQ archives first");
return false;
}
// Set dynamic file cache budget based on available RAM.
// Bias toward MPQ decompressed-file caching to minimize runtime read/decompress stalls.
if (!manifest_.load(manifestPath)) {
LOG_ERROR("Failed to load manifest");
return false;
}
initialized = true;
LOG_INFO("Asset manager initialized: ", manifest_.getEntryCount(),
" files indexed (file cache: ", fileCacheBudget / (1024 * 1024), " MB)");
return true;
}
void AssetManager::setupFileCacheBudget() {
auto& memMonitor = core::MemoryMonitor::getInstance();
size_t recommendedBudget = memMonitor.getRecommendedCacheBudget();
size_t dynamicBudget = (recommendedBudget * 3) / 4; // 75% of global cache budget
size_t dynamicBudget = (recommendedBudget * 3) / 4;
// The MemoryMonitor recommendation is intentionally aggressive; without a cap, large patch MPQs
// can cause the decompressed-file cache to balloon into tens/hundreds of GB and push the OS into swap.
// Provide env overrides and clamp to a safe default.
const size_t envFixedMB = parseEnvSizeMB("WOWEE_FILE_CACHE_MB");
const size_t envMaxMB = parseEnvSizeMB("WOWEE_FILE_CACHE_MAX_MB");
const size_t minBudgetBytes = 256ull * 1024ull * 1024ull; // 256 MB
const size_t defaultMaxBudgetBytes = 32768ull * 1024ull * 1024ull; // 32 GB
const size_t minBudgetBytes = 256ull * 1024ull * 1024ull;
const size_t defaultMaxBudgetBytes = 32768ull * 1024ull * 1024ull;
const size_t maxBudgetBytes = (envMaxMB > 0)
? (envMaxMB * 1024ull * 1024ull)
: defaultMaxBudgetBytes;
@ -74,13 +87,6 @@ bool AssetManager::initialize(const std::string& dataPath_) {
} else {
fileCacheBudget = std::clamp(dynamicBudget, minBudgetBytes, maxBudgetBytes);
}
initialized = true;
LOG_INFO("Asset manager initialized (file cache: ",
fileCacheBudget / (1024 * 1024), " MB, recommended=",
recommendedBudget / (1024 * 1024), " MB, max=",
maxBudgetBytes / (1024 * 1024), " MB)");
return true;
}
void AssetManager::shutdown() {
@ -90,7 +96,6 @@ void AssetManager::shutdown() {
LOG_INFO("Shutting down asset manager");
// Log cache statistics
if (fileCacheHits + fileCacheMisses > 0) {
float hitRate = (float)fileCacheHits / (fileCacheHits + fileCacheMisses) * 100.0f;
LOG_INFO("File cache stats: ", fileCacheHits, " hits, ", fileCacheMisses, " misses (",
@ -98,8 +103,6 @@ void AssetManager::shutdown() {
}
clearCache();
mpqManager.shutdown();
initialized = false;
}
@ -109,19 +112,22 @@ BLPImage AssetManager::loadTexture(const std::string& path) {
return BLPImage();
}
// Normalize path
std::string normalizedPath = normalizePath(path);
LOG_DEBUG("Loading texture: ", normalizedPath);
// Route through readFile() so decompressed MPQ bytes participate in file cache.
// Check for PNG override
BLPImage pngImage = tryLoadPngOverride(normalizedPath);
if (pngImage.isValid()) {
return pngImage;
}
std::vector<uint8_t> blpData = readFile(normalizedPath);
if (blpData.empty()) {
LOG_WARNING("Texture not found: ", normalizedPath);
return BLPImage();
}
// Load BLP
BLPImage image = BLPLoader::load(blpData);
if (!image.isValid()) {
LOG_ERROR("Failed to load texture: ", normalizedPath);
@ -132,13 +138,47 @@ BLPImage AssetManager::loadTexture(const std::string& path) {
return image;
}
BLPImage AssetManager::tryLoadPngOverride(const std::string& normalizedPath) const {
if (normalizedPath.size() < 4) return BLPImage();
std::string ext = normalizedPath.substr(normalizedPath.size() - 4);
if (ext != ".blp") return BLPImage();
std::string fsPath = manifest_.resolveFilesystemPath(normalizedPath);
if (fsPath.empty()) return BLPImage();
// Replace .blp/.BLP extension with .png
std::string pngPath = fsPath.substr(0, fsPath.size() - 4) + ".png";
if (!LooseFileReader::fileExists(pngPath)) {
return BLPImage();
}
int w, h, channels;
unsigned char* pixels = stbi_load(pngPath.c_str(), &w, &h, &channels, 4);
if (!pixels) {
LOG_WARNING("PNG override exists but failed to load: ", pngPath);
return BLPImage();
}
BLPImage image;
image.width = w;
image.height = h;
image.channels = 4;
image.format = BLPFormat::BLP2;
image.compression = BLPCompression::ARGB8888;
image.data.assign(pixels, pixels + (w * h * 4));
stbi_image_free(pixels);
LOG_INFO("PNG override loaded: ", pngPath, " (", w, "x", h, ")");
return image;
}
std::shared_ptr<DBCFile> AssetManager::loadDBC(const std::string& name) {
if (!initialized) {
LOG_ERROR("AssetManager not initialized");
return nullptr;
}
// Check cache first
auto it = dbcCache.find(name);
if (it != dbcCache.end()) {
LOG_DEBUG("DBC already loaded (cached): ", name);
@ -147,28 +187,20 @@ std::shared_ptr<DBCFile> AssetManager::loadDBC(const std::string& name) {
LOG_DEBUG("Loading DBC: ", name);
// Construct DBC path (DBFilesClient directory)
std::string dbcPath = "DBFilesClient\\" + name;
// Read DBC file from MPQ (must hold readMutex — StormLib is not thread-safe)
std::vector<uint8_t> dbcData;
{
std::lock_guard<std::mutex> lock(readMutex);
dbcData = mpqManager.readFile(dbcPath);
}
std::vector<uint8_t> dbcData = readFile(dbcPath);
if (dbcData.empty()) {
LOG_WARNING("DBC not found: ", dbcPath);
return nullptr;
}
// Load DBC
auto dbc = std::make_shared<DBCFile>();
if (!dbc->load(dbcData)) {
LOG_ERROR("Failed to load DBC: ", dbcPath);
return nullptr;
}
// Cache the DBC
dbcCache[name] = dbc;
LOG_INFO("Loaded DBC: ", name, " (", dbc->getRecordCount(), " records)");
@ -187,57 +219,45 @@ bool AssetManager::fileExists(const std::string& path) const {
if (!initialized) {
return false;
}
std::lock_guard<std::mutex> lock(readMutex);
return mpqManager.fileExists(normalizePath(path));
return manifest_.hasEntry(normalizePath(path));
}
std::vector<uint8_t> AssetManager::readFile(const std::string& path) const {
if (!initialized) {
return std::vector<uint8_t>();
return {};
}
std::string normalized = normalizePath(path);
// Check cache first
{
std::lock_guard<std::mutex> cacheLock(cacheMutex);
// Check cache first
auto it = fileCache.find(normalized);
if (it != fileCache.end()) {
// Cache hit - update access time and return cached data
it->second.lastAccessTime = ++fileCacheAccessCounter;
fileCacheHits++;
return it->second.data;
}
}
// Cache miss path: serialize MPQ reads. Before reading, re-check cache while holding
// readMutex so only one thread performs decompression per hot path at a time.
std::vector<uint8_t> data;
{
std::lock_guard<std::mutex> readLock(readMutex);
{
std::lock_guard<std::mutex> cacheLock(cacheMutex);
auto it = fileCache.find(normalized);
if (it != fileCache.end()) {
it->second.lastAccessTime = ++fileCacheAccessCounter;
fileCacheHits++;
return it->second.data;
}
fileCacheMisses++;
}
data = mpqManager.readFile(normalized);
// Read from filesystem (fully parallel, no serialization needed)
std::string fsPath = manifest_.resolveFilesystemPath(normalized);
if (fsPath.empty()) {
return {};
}
auto data = LooseFileReader::readFile(fsPath);
if (data.empty()) {
return data; // File not found
LOG_WARNING("Manifest entry exists but file unreadable: ", fsPath);
return data;
}
// Add to cache if within budget
size_t fileSize = data.size();
if (fileSize > 0 && fileSize < fileCacheBudget / 2) { // Don't cache files > 50% of budget (very aggressive)
if (fileSize > 0 && fileSize < fileCacheBudget / 2) {
std::lock_guard<std::mutex> cacheLock(cacheMutex);
// Evict old entries if needed (LRU)
while (fileCacheTotalBytes + fileSize > fileCacheBudget && !fileCache.empty()) {
// Find least recently used entry
auto lru = fileCache.begin();
for (auto it = fileCache.begin(); it != fileCache.end(); ++it) {
if (it->second.lastAccessTime < lru->second.lastAccessTime) {
@ -248,7 +268,6 @@ std::vector<uint8_t> AssetManager::readFile(const std::string& path) const {
fileCache.erase(lru);
}
// Add new entry
CachedFile cached;
cached.data = data;
cached.lastAccessTime = ++fileCacheAccessCounter;
@ -261,18 +280,16 @@ std::vector<uint8_t> AssetManager::readFile(const std::string& path) const {
std::vector<uint8_t> AssetManager::readFileOptional(const std::string& path) const {
if (!initialized) {
return std::vector<uint8_t>();
return {};
}
// Avoid MPQManager missing-file warnings for expected probe misses.
if (!fileExists(path)) {
return std::vector<uint8_t>();
return {};
}
return readFile(path);
}
void AssetManager::clearCache() {
std::scoped_lock lock(readMutex, cacheMutex);
std::lock_guard<std::mutex> lock(cacheMutex);
dbcCache.clear();
fileCache.clear();
fileCacheTotalBytes = 0;
@ -282,13 +299,9 @@ void AssetManager::clearCache() {
std::string AssetManager::normalizePath(const std::string& path) const {
std::string normalized = path;
// Convert forward slashes to backslashes (WoW uses backslashes)
std::replace(normalized.begin(), normalized.end(), '/', '\\');
// Lowercase for case-insensitive cache keys (improves hit rate across mixed-case callers).
std::transform(normalized.begin(), normalized.end(), normalized.begin(),
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return normalized;
}

97
src/pipeline/asset_manifest.cpp Normal file
View file

@ -0,0 +1,97 @@
#include "pipeline/asset_manifest.hpp"
#include "core/logger.hpp"
#include <nlohmann/json.hpp>
#include <fstream>
#include <filesystem>
#include <chrono>
namespace wowee {
namespace pipeline {
bool AssetManifest::load(const std::string& manifestPath) {
auto startTime = std::chrono::steady_clock::now();
std::ifstream file(manifestPath);
if (!file.is_open()) {
LOG_ERROR("Failed to open manifest: ", manifestPath);
return false;
}
nlohmann::json doc;
try {
doc = nlohmann::json::parse(file);
} catch (const nlohmann::json::parse_error& e) {
LOG_ERROR("Failed to parse manifest JSON: ", e.what());
return false;
}
// Read header
int version = doc.value("version", 0);
if (version != 1) {
LOG_ERROR("Unsupported manifest version: ", version);
return false;
}
basePath_ = doc.value("basePath", "assets");
manifestDir_ = std::filesystem::path(manifestPath).parent_path().string();
// If basePath is relative, resolve against manifest directory
if (!basePath_.empty() && basePath_[0] != '/') {
basePath_ = manifestDir_ + "/" + basePath_;
}
// Parse entries
auto& entriesObj = doc["entries"];
if (!entriesObj.is_object()) {
LOG_ERROR("Manifest missing 'entries' object");
return false;
}
entries_.reserve(entriesObj.size());
for (auto& [key, val] : entriesObj.items()) {
Entry entry;
entry.filesystemPath = val.value("p", "");
entry.size = val.value("s", uint64_t(0));
// Parse CRC32 from hex string
std::string hexHash = val.value("h", "");
if (!hexHash.empty()) {
entry.crc32 = static_cast<uint32_t>(std::strtoul(hexHash.c_str(), nullptr, 16));
} else {
entry.crc32 = 0;
}
entries_[key] = std::move(entry);
}
loaded_ = true;
auto elapsed = std::chrono::steady_clock::now() - startTime;
auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count();
LOG_INFO("Loaded asset manifest: ", entries_.size(), " entries in ", ms, "ms (base: ", basePath_, ")");
return true;
}
const AssetManifest::Entry* AssetManifest::lookup(const std::string& normalizedWowPath) const {
auto it = entries_.find(normalizedWowPath);
if (it != entries_.end()) {
return &it->second;
}
return nullptr;
}
std::string AssetManifest::resolveFilesystemPath(const std::string& normalizedWowPath) const {
auto it = entries_.find(normalizedWowPath);
if (it == entries_.end()) {
return {};
}
return basePath_ + "/" + it->second.filesystemPath;
}
bool AssetManifest::hasEntry(const std::string& normalizedWowPath) const {
return entries_.find(normalizedWowPath) != entries_.end();
}
} // namespace pipeline
} // namespace wowee

47
src/pipeline/loose_file_reader.cpp Normal file
View file

@ -0,0 +1,47 @@
#include "pipeline/loose_file_reader.hpp"
#include "core/logger.hpp"
#include <fstream>
#include <filesystem>
namespace wowee {
namespace pipeline {
std::vector<uint8_t> LooseFileReader::readFile(const std::string& filesystemPath) {
std::ifstream file(filesystemPath, std::ios::binary | std::ios::ate);
if (!file.is_open()) {
return {};
}
auto size = file.tellg();
if (size <= 0) {
return {};
}
std::vector<uint8_t> data(static_cast<size_t>(size));
file.seekg(0, std::ios::beg);
file.read(reinterpret_cast<char*>(data.data()), size);
if (!file.good()) {
LOG_WARNING("Incomplete read of: ", filesystemPath);
data.resize(static_cast<size_t>(file.gcount()));
}
return data;
}
bool LooseFileReader::fileExists(const std::string& filesystemPath) {
std::error_code ec;
return std::filesystem::exists(filesystemPath, ec);
}
uint64_t LooseFileReader::getFileSize(const std::string& filesystemPath) {
std::error_code ec;
auto size = std::filesystem::file_size(filesystemPath, ec);
if (ec) {
return 0;
}
return static_cast<uint64_t>(size);
}
} // namespace pipeline
} // namespace wowee

2
src/rendering/minimap.cpp
View file

@ -267,7 +267,7 @@ void Minimap::parseTRS() {
if (trsParsed || !assetManager) return;
trsParsed = true;
auto data = assetManager->getMPQManager().readFile("Textures\\Minimap\\md5translate.trs");
auto data = assetManager->readFile("Textures\\Minimap\\md5translate.trs");
if (data.empty()) {
LOG_WARNING("Failed to load md5translate.trs");
return;

380
tools/asset_extract/extractor.cpp Normal file
View file

@ -0,0 +1,380 @@
#include "extractor.hpp"
#include "path_mapper.hpp"
#include "manifest_writer.hpp"
#include <StormLib.h>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <mutex>
#include <set>
#include <thread>
#include <vector>
#ifndef INVALID_HANDLE_VALUE
#define INVALID_HANDLE_VALUE ((HANDLE)(long long)-1)
#endif
namespace wowee {
namespace tools {
namespace fs = std::filesystem;
// Archive descriptor for priority-based loading
struct ArchiveDesc {
std::string path;
int priority;
};
static std::string toLowerStr(std::string s) {
std::transform(s.begin(), s.end(), s.begin(),
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return s;
}
static std::string normalizeWowPath(const std::string& path) {
std::string n = path;
std::replace(n.begin(), n.end(), '/', '\\');
std::transform(n.begin(), n.end(), n.begin(),
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return n;
}
// Discover archive files in the same priority order as MPQManager
static std::vector<ArchiveDesc> discoverArchives(const std::string& mpqDir) {
std::vector<ArchiveDesc> result;
auto tryAdd = [&](const std::string& name, int prio) {
std::string fullPath = mpqDir + "/" + name;
if (fs::exists(fullPath)) {
result.push_back({fullPath, prio});
}
};
// Base archives (priority 100)
tryAdd("common.MPQ", 100);
tryAdd("common-2.MPQ", 100);
tryAdd("expansion.MPQ", 100);
tryAdd("lichking.MPQ", 100);
// Patch archives (priority 150-500)
tryAdd("patch.MPQ", 150);
tryAdd("patch-2.MPQ", 200);
tryAdd("patch-3.MPQ", 300);
tryAdd("patch-4.MPQ", 400);
tryAdd("patch-5.MPQ", 500);
// Letter patches (priority 800-925)
for (char c = 'a'; c <= 'z'; ++c) {
std::string lower = std::string("patch-") + c + ".mpq";
std::string upper = std::string("Patch-") + static_cast<char>(std::toupper(c)) + ".mpq";
int prioLower = 800 + (c - 'a');
int prioUpper = 900 + (c - 'a');
tryAdd(lower, prioLower);
tryAdd(upper, prioUpper);
}
// Locale archives
tryAdd("enUS/backup-enUS.MPQ", 230);
tryAdd("enUS/base-enUS.MPQ", 235);
tryAdd("enUS/speech-enUS.MPQ", 240);
tryAdd("enUS/expansion-speech-enUS.MPQ", 245);
tryAdd("enUS/expansion-locale-enUS.MPQ", 246);
tryAdd("enUS/lichking-speech-enUS.MPQ", 248);
tryAdd("enUS/lichking-locale-enUS.MPQ", 249);
tryAdd("enUS/locale-enUS.MPQ", 250);
tryAdd("enUS/patch-enUS.MPQ", 450);
tryAdd("enUS/patch-enUS-2.MPQ", 460);
tryAdd("enUS/patch-enUS-3.MPQ", 470);
// Sort by priority so highest-priority archives are last
// (we'll iterate highest-prio first when extracting)
std::sort(result.begin(), result.end(),
[](const ArchiveDesc& a, const ArchiveDesc& b) { return a.priority < b.priority; });
return result;
}
bool Extractor::enumerateFiles(const Options& opts,
std::vector<std::string>& outFiles) {
// Open all archives, enumerate files from highest priority to lowest.
// Use a set to deduplicate (highest-priority version wins).
auto archives = discoverArchives(opts.mpqDir);
if (archives.empty()) {
std::cerr << "No MPQ archives found in: " << opts.mpqDir << "\n";
return false;
}
std::cout << "Found " << archives.size() << " MPQ archives\n";
// Enumerate from highest priority first so first-seen files win
std::set<std::string> seenNormalized;
std::vector<std::pair<std::string, std::string>> fileList; // (original name, archive path)
for (auto it = archives.rbegin(); it != archives.rend(); ++it) {
HANDLE hMpq = nullptr;
if (!SFileOpenArchive(it->path.c_str(), 0, 0, &hMpq)) {
std::cerr << " Failed to open: " << it->path << "\n";
continue;
}
if (opts.verbose) {
std::cout << " Scanning: " << it->path << " (priority " << it->priority << ")\n";
}
SFILE_FIND_DATA findData;
HANDLE hFind = SFileFindFirstFile(hMpq, "*", &findData, nullptr);
if (hFind) {
do {
std::string fileName = findData.cFileName;
// Skip internal listfile/attributes
if (fileName == "(listfile)" || fileName == "(attributes)" ||
fileName == "(signature)" || fileName == "(patch_metadata)") {
continue;
}
std::string norm = normalizeWowPath(fileName);
if (seenNormalized.insert(norm).second) {
// First time seeing this file — this is the highest-priority version
outFiles.push_back(fileName);
}
} while (SFileFindNextFile(hFind, &findData));
SFileFindClose(hFind);
}
SFileCloseArchive(hMpq);
}
std::cout << "Enumerated " << outFiles.size() << " unique files\n";
return true;
}
bool Extractor::run(const Options& opts) {
auto startTime = std::chrono::steady_clock::now();
// Enumerate all unique files across all archives
std::vector<std::string> files;
if (!enumerateFiles(opts, files)) {
return false;
}
if (files.empty()) {
std::cerr << "No files to extract\n";
return false;
}
// Create output directory
std::error_code ec;
fs::create_directories(opts.outputDir, ec);
if (ec) {
std::cerr << "Failed to create output directory: " << ec.message() << "\n";
return false;
}
auto archives = discoverArchives(opts.mpqDir);
// Determine thread count
int numThreads = opts.threads;
if (numThreads <= 0) {
numThreads = static_cast<int>(std::thread::hardware_concurrency());
if (numThreads <= 0) numThreads = 4;
}
Stats stats;
std::mutex manifestMutex;
std::vector<ManifestWriter::FileEntry> manifestEntries;
// Partition files across threads
std::atomic<size_t> fileIndex{0};
size_t totalFiles = files.size();
auto workerFn = [&]() {
// Each thread opens ALL archives independently (StormLib is not thread-safe per handle).
// Sorted highest-priority last, so we iterate in reverse to find the winning version.
struct ThreadArchive {
HANDLE handle;
int priority;
};
std::vector<ThreadArchive> threadHandles;
for (const auto& ad : archives) {
HANDLE h = nullptr;
if (SFileOpenArchive(ad.path.c_str(), 0, 0, &h)) {
threadHandles.push_back({h, ad.priority});
}
}
if (threadHandles.empty()) {
std::cerr << "Worker thread: failed to open any archives\n";
return;
}
while (true) {
size_t idx = fileIndex.fetch_add(1);
if (idx >= totalFiles) break;
const std::string& wowPath = files[idx];
std::string normalized = normalizeWowPath(wowPath);
// Map to new filesystem path
std::string mappedPath = PathMapper::mapPath(wowPath);
std::string fullOutputPath = opts.outputDir + "/" + mappedPath;
// Search archives in reverse priority order (highest priority first)
HANDLE hFile = nullptr;
for (auto it = threadHandles.rbegin(); it != threadHandles.rend(); ++it) {
if (SFileOpenFileEx(it->handle, wowPath.c_str(), 0, &hFile)) {
break;
}
hFile = nullptr;
}
if (!hFile) {
stats.filesFailed++;
continue;
}
DWORD fileSize = SFileGetFileSize(hFile, nullptr);
if (fileSize == SFILE_INVALID_SIZE || fileSize == 0) {
SFileCloseFile(hFile);
stats.filesSkipped++;
continue;
}
std::vector<uint8_t> data(fileSize);
DWORD bytesRead = 0;
if (!SFileReadFile(hFile, data.data(), fileSize, &bytesRead, nullptr)) {
SFileCloseFile(hFile);
stats.filesFailed++;
continue;
}
SFileCloseFile(hFile);
data.resize(bytesRead);
// Create output directory
fs::path outPath(fullOutputPath);
fs::create_directories(outPath.parent_path(), ec);
// Write file
std::ofstream out(fullOutputPath, std::ios::binary);
if (!out.is_open()) {
stats.filesFailed++;
continue;
}
out.write(reinterpret_cast<const char*>(data.data()), data.size());
out.close();
// Compute CRC32
uint32_t crc = ManifestWriter::computeCRC32(data.data(), data.size());
// Add manifest entry
ManifestWriter::FileEntry entry;
entry.wowPath = normalized;
entry.filesystemPath = mappedPath;
entry.size = data.size();
entry.crc32 = crc;
{
std::lock_guard<std::mutex> lock(manifestMutex);
manifestEntries.push_back(std::move(entry));
}
stats.filesExtracted++;
stats.bytesExtracted += data.size();
// Progress
uint64_t done = stats.filesExtracted.load();
if (done % 1000 == 0) {
std::cout << "\r Extracted " << done << " / " << totalFiles << " files..."
<< std::flush;
}
}
for (auto& th : threadHandles) {
SFileCloseArchive(th.handle);
}
};
std::cout << "Extracting " << totalFiles << " files using " << numThreads << " threads...\n";
std::vector<std::thread> threads;
for (int i = 0; i < numThreads; ++i) {
threads.emplace_back(workerFn);
}
for (auto& t : threads) {
t.join();
}
std::cout << "\r Extracted " << stats.filesExtracted.load() << " files ("
<< stats.bytesExtracted.load() / (1024 * 1024) << " MB), "
<< stats.filesSkipped.load() << " skipped, "
<< stats.filesFailed.load() << " failed\n";
// Sort manifest entries for deterministic output
std::sort(manifestEntries.begin(), manifestEntries.end(),
[](const ManifestWriter::FileEntry& a, const ManifestWriter::FileEntry& b) {
return a.wowPath < b.wowPath;
});
// Write manifest
std::string manifestPath = opts.outputDir + "/manifest.json";
// basePath is "." since manifest sits inside the output directory
if (!ManifestWriter::write(manifestPath, ".", manifestEntries)) {
std::cerr << "Failed to write manifest: " << manifestPath << "\n";
return false;
}
std::cout << "Wrote manifest: " << manifestPath << " (" << manifestEntries.size() << " entries)\n";
// Verification pass
if (opts.verify) {
std::cout << "Verifying extracted files...\n";
uint64_t verified = 0, verifyFailed = 0;
for (const auto& entry : manifestEntries) {
std::string fsPath = opts.outputDir + "/" + entry.filesystemPath;
std::ifstream f(fsPath, std::ios::binary | std::ios::ate);
if (!f.is_open()) {
std::cerr << " MISSING: " << fsPath << "\n";
verifyFailed++;
continue;
}
auto size = f.tellg();
if (static_cast<uint64_t>(size) != entry.size) {
std::cerr << " SIZE MISMATCH: " << fsPath << " (expected "
<< entry.size << ", got " << size << ")\n";
verifyFailed++;
continue;
}
f.seekg(0);
std::vector<uint8_t> data(static_cast<size_t>(size));
f.read(reinterpret_cast<char*>(data.data()), size);
uint32_t crc = ManifestWriter::computeCRC32(data.data(), data.size());
if (crc != entry.crc32) {
std::cerr << " CRC MISMATCH: " << fsPath << "\n";
verifyFailed++;
continue;
}
verified++;
}
std::cout << "Verified " << verified << " files";
if (verifyFailed > 0) {
std::cout << " (" << verifyFailed << " FAILED)";
}
std::cout << "\n";
}
auto elapsed = std::chrono::steady_clock::now() - startTime;
auto secs = std::chrono::duration_cast<std::chrono::seconds>(elapsed).count();
std::cout << "Done in " << secs / 60 << "m " << secs % 60 << "s\n";
return true;
}
} // namespace tools
} // namespace wowee

43
tools/asset_extract/extractor.hpp Normal file
View file

@ -0,0 +1,43 @@
#pragma once
#include <string>
#include <vector>
#include <atomic>
#include <cstdint>
namespace wowee {
namespace tools {
/**
* Extraction pipeline: MPQ archives → loose files + manifest
*/
class Extractor {
public:
struct Options {
std::string mpqDir; // Path to WoW Data directory
std::string outputDir; // Output directory for extracted assets
int threads = 0; // 0 = auto-detect
bool verify = false; // CRC32 verify after extraction
bool verbose = false; // Verbose logging
};
struct Stats {
std::atomic<uint64_t> filesExtracted{0};
std::atomic<uint64_t> bytesExtracted{0};
std::atomic<uint64_t> filesSkipped{0};
std::atomic<uint64_t> filesFailed{0};
};
/**
* Run the extraction pipeline
* @return true on success
*/
static bool run(const Options& opts);
private:
static bool enumerateFiles(const Options& opts,
std::vector<std::string>& outFiles);
};
} // namespace tools
} // namespace wowee

62
tools/asset_extract/main.cpp Normal file
View file

@ -0,0 +1,62 @@
#include "extractor.hpp"
#include <iostream>
#include <string>
#include <cstring>
static void printUsage(const char* prog) {
std::cout << "Usage: " << prog << " --mpq-dir <path> --output <path> [options]\n"
<< "\n"
<< "Extract WoW MPQ archives to organized loose files with manifest.\n"
<< "\n"
<< "Required:\n"
<< " --mpq-dir <path> Path to WoW Data directory containing MPQ files\n"
<< " --output <path> Output directory for extracted assets\n"
<< "\n"
<< "Options:\n"
<< " --verify CRC32 verify all extracted files\n"
<< " --threads <N> Number of extraction threads (default: auto)\n"
<< " --verbose Verbose output\n"
<< " --help Show this help\n";
}
int main(int argc, char** argv) {
wowee::tools::Extractor::Options opts;
for (int i = 1; i < argc; ++i) {
if (std::strcmp(argv[i], "--mpq-dir") == 0 && i + 1 < argc) {
opts.mpqDir = argv[++i];
} else if (std::strcmp(argv[i], "--output") == 0 && i + 1 < argc) {
opts.outputDir = argv[++i];
} else if (std::strcmp(argv[i], "--threads") == 0 && i + 1 < argc) {
opts.threads = std::atoi(argv[++i]);
} else if (std::strcmp(argv[i], "--verify") == 0) {
opts.verify = true;
} else if (std::strcmp(argv[i], "--verbose") == 0) {
opts.verbose = true;
} else if (std::strcmp(argv[i], "--help") == 0 || std::strcmp(argv[i], "-h") == 0) {
printUsage(argv[0]);
return 0;
} else {
std::cerr << "Unknown option: " << argv[i] << "\n";
printUsage(argv[0]);
return 1;
}
}
if (opts.mpqDir.empty() || opts.outputDir.empty()) {
std::cerr << "Error: --mpq-dir and --output are required\n\n";
printUsage(argv[0]);
return 1;
}
std::cout << "=== Wowee Asset Extractor ===\n";
std::cout << "MPQ directory: " << opts.mpqDir << "\n";
std::cout << "Output: " << opts.outputDir << "\n";
if (!wowee::tools::Extractor::run(opts)) {
std::cerr << "Extraction failed!\n";
return 1;
}
return 0;
}

67
tools/asset_extract/manifest_writer.cpp Normal file
View file

@ -0,0 +1,67 @@
#include "manifest_writer.hpp"
#include <fstream>
#include <sstream>
#include <iomanip>
#include <zlib.h>
namespace wowee {
namespace tools {
uint32_t ManifestWriter::computeCRC32(const uint8_t* data, size_t size) {
return static_cast<uint32_t>(::crc32(::crc32(0L, Z_NULL, 0), data, static_cast<uInt>(size)));
}
bool ManifestWriter::write(const std::string& outputPath,
const std::string& basePath,
const std::vector<FileEntry>& entries) {
// Write JSON manually to avoid pulling nlohmann/json into the tool
// (though it would also work fine). This keeps the tool dependency-light.
std::ofstream file(outputPath);
if (!file.is_open()) {
return false;
}
file << "{\n";
file << " \"version\": 1,\n";
file << " \"basePath\": \"" << basePath << "\",\n";
file << " \"fileCount\": " << entries.size() << ",\n";
file << " \"entries\": {\n";
for (size_t i = 0; i < entries.size(); ++i) {
const auto& e = entries[i];
// Escape backslashes in WoW path for JSON
std::string escapedKey;
for (char c : e.wowPath) {
if (c == '\\') escapedKey += "\\\\";
else if (c == '"') escapedKey += "\\\"";
else escapedKey += c;
}
std::string escapedPath;
for (char c : e.filesystemPath) {
if (c == '\\') escapedPath += "\\\\";
else if (c == '"') escapedPath += "\\\"";
else escapedPath += c;
}
// CRC32 as hex
std::ostringstream hexCrc;
hexCrc << std::hex << std::setfill('0') << std::setw(8) << e.crc32;
file << " \"" << escapedKey << "\": {\"p\": \"" << escapedPath
<< "\", \"s\": " << e.size
<< ", \"h\": \"" << hexCrc.str() << "\"}";
if (i + 1 < entries.size()) file << ",";
file << "\n";
}
file << " }\n";
file << "}\n";
return file.good();
}
} // namespace tools
} // namespace wowee

40
tools/asset_extract/manifest_writer.hpp Normal file
View file

@ -0,0 +1,40 @@
#pragma once
#include <string>
#include <vector>
#include <cstdint>
namespace wowee {
namespace tools {
/**
* Generates manifest.json from extracted file metadata.
*/
class ManifestWriter {
public:
struct FileEntry {
std::string wowPath; // Normalized WoW virtual path (lowercase, backslash)
std::string filesystemPath; // Relative path from basePath (forward slashes, original case)
uint64_t size; // File size in bytes
uint32_t crc32; // CRC32 checksum
};
/**
* Write manifest.json
* @param outputPath Full path to manifest.json
* @param basePath Value for basePath field (e.g., "assets")
* @param entries All extracted file entries
* @return true on success
*/
static bool write(const std::string& outputPath,
const std::string& basePath,
const std::vector<FileEntry>& entries);
/**
* Compute CRC32 of file data
*/
static uint32_t computeCRC32(const uint8_t* data, size_t size);
};
} // namespace tools
} // namespace wowee

207
tools/asset_extract/path_mapper.cpp Normal file
View file

@ -0,0 +1,207 @@
#include "path_mapper.hpp"
#include <algorithm>
#include <cctype>
namespace wowee {
namespace tools {
std::string PathMapper::toLower(const std::string& str) {
std::string result = str;
std::transform(result.begin(), result.end(), result.begin(),
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return result;
}
std::string PathMapper::toForwardSlash(const std::string& str) {
std::string result = str;
std::replace(result.begin(), result.end(), '\\', '/');
return result;
}
bool PathMapper::startsWithCI(const std::string& str, const std::string& prefix) {
if (str.size() < prefix.size()) return false;
for (size_t i = 0; i < prefix.size(); ++i) {
if (std::tolower(static_cast<unsigned char>(str[i])) !=
std::tolower(static_cast<unsigned char>(prefix[i]))) {
return false;
}
}
return true;
}
std::string PathMapper::extractAfterPrefix(const std::string& path, size_t prefixLen) {
if (prefixLen >= path.size()) return {};
return path.substr(prefixLen);
}
std::string PathMapper::mapPath(const std::string& wowPath) {
// Preserve original casing in the remainder for filesystem readability
std::string rest;
// DBFilesClient\ → db/
if (startsWithCI(wowPath, "DBFilesClient\\")) {
rest = extractAfterPrefix(wowPath, 14);
return "db/" + toForwardSlash(rest);
}
// Character\{Race}\{Gender}\ → character/{race}/{gender}/
if (startsWithCI(wowPath, "Character\\")) {
rest = extractAfterPrefix(wowPath, 10);
std::string lowered = toLower(rest);
return "character/" + toForwardSlash(lowered);
}
// Creature\{Name}\ → creature/{name}/
if (startsWithCI(wowPath, "Creature\\")) {
rest = extractAfterPrefix(wowPath, 9);
// Keep first component lowercase for directory, preserve filename case
std::string fwd = toForwardSlash(rest);
auto slash = fwd.find('/');
if (slash != std::string::npos) {
return "creature/" + toLower(fwd.substr(0, slash)) + "/" + fwd.substr(slash + 1);
}
return "creature/" + fwd;
}
// Item\ObjectComponents\{Type}\ → item/{type}/
if (startsWithCI(wowPath, "Item\\ObjectComponents\\")) {
rest = extractAfterPrefix(wowPath, 22);
std::string fwd = toForwardSlash(rest);
auto slash = fwd.find('/');
if (slash != std::string::npos) {
return "item/" + toLower(fwd.substr(0, slash)) + "/" + fwd.substr(slash + 1);
}
return "item/" + fwd;
}
// Item\TextureComponents\ → item/texture/
if (startsWithCI(wowPath, "Item\\TextureComponents\\")) {
rest = extractAfterPrefix(wowPath, 23);
return "item/texture/" + toForwardSlash(rest);
}
// Interface\Icons\ → interface/icons/
if (startsWithCI(wowPath, "Interface\\Icons\\")) {
rest = extractAfterPrefix(wowPath, 16);
return "interface/icons/" + toForwardSlash(rest);
}
// Interface\GossipFrame\ → interface/gossip/
if (startsWithCI(wowPath, "Interface\\GossipFrame\\")) {
rest = extractAfterPrefix(wowPath, 21);
return "interface/gossip/" + toForwardSlash(rest);
}
// Interface\{rest} → interface/{rest}/
if (startsWithCI(wowPath, "Interface\\")) {
rest = extractAfterPrefix(wowPath, 10);
return "interface/" + toForwardSlash(rest);
}
// Textures\Minimap\ → terrain/minimap/
if (startsWithCI(wowPath, "Textures\\Minimap\\")) {
rest = extractAfterPrefix(wowPath, 17);
return "terrain/minimap/" + toForwardSlash(rest);
}
// Textures\BakedNpcTextures\ → creature/baked/
if (startsWithCI(wowPath, "Textures\\BakedNpcTextures\\")) {
rest = extractAfterPrefix(wowPath, 25);
return "creature/baked/" + toForwardSlash(rest);
}
// Textures\{rest} → terrain/textures/{rest}
if (startsWithCI(wowPath, "Textures\\")) {
rest = extractAfterPrefix(wowPath, 9);
return "terrain/textures/" + toForwardSlash(rest);
}
// World\Maps\{Map}\ → terrain/maps/{map}/
if (startsWithCI(wowPath, "World\\Maps\\")) {
rest = extractAfterPrefix(wowPath, 11);
std::string fwd = toForwardSlash(rest);
auto slash = fwd.find('/');
if (slash != std::string::npos) {
return "terrain/maps/" + toLower(fwd.substr(0, slash)) + "/" + fwd.substr(slash + 1);
}
return "terrain/maps/" + fwd;
}
// World\wmo\ → world/wmo/ (preserve subpath)
if (startsWithCI(wowPath, "World\\wmo\\")) {
rest = extractAfterPrefix(wowPath, 10);
return "world/wmo/" + toForwardSlash(rest);
}
// World\Doodads\ → world/doodads/
if (startsWithCI(wowPath, "World\\Doodads\\")) {
rest = extractAfterPrefix(wowPath, 14);
return "world/doodads/" + toForwardSlash(rest);
}
// World\{rest} → world/{rest}/
if (startsWithCI(wowPath, "World\\")) {
rest = extractAfterPrefix(wowPath, 6);
return "world/" + toForwardSlash(rest);
}
// Environments\ → environment/
if (startsWithCI(wowPath, "Environments\\")) {
rest = extractAfterPrefix(wowPath, 13);
return "environment/" + toForwardSlash(rest);
}
// Sound\Ambience\ → sound/ambient/
if (startsWithCI(wowPath, "Sound\\Ambience\\")) {
rest = extractAfterPrefix(wowPath, 15);
return "sound/ambient/" + toForwardSlash(rest);
}
// Sound\Character\ → sound/character/
if (startsWithCI(wowPath, "Sound\\Character\\")) {
rest = extractAfterPrefix(wowPath, 16);
return "sound/character/" + toForwardSlash(rest);
}
// Sound\Doodad\ → sound/doodad/
if (startsWithCI(wowPath, "Sound\\Doodad\\")) {
rest = extractAfterPrefix(wowPath, 13);
return "sound/doodad/" + toForwardSlash(rest);
}
// Sound\Creature\ → sound/creature/
if (startsWithCI(wowPath, "Sound\\Creature\\")) {
rest = extractAfterPrefix(wowPath, 15);
return "sound/creature/" + toForwardSlash(rest);
}
// Sound\Spells\ → sound/spell/
if (startsWithCI(wowPath, "Sound\\Spells\\")) {
rest = extractAfterPrefix(wowPath, 13);
return "sound/spell/" + toForwardSlash(rest);
}
// Sound\Music\ → sound/music/
if (startsWithCI(wowPath, "Sound\\Music\\")) {
rest = extractAfterPrefix(wowPath, 12);
return "sound/music/" + toForwardSlash(rest);
}
// Sound\{rest} → sound/{rest}/
if (startsWithCI(wowPath, "Sound\\")) {
rest = extractAfterPrefix(wowPath, 6);
return "sound/" + toForwardSlash(rest);
}
// Spells\ → spell/
if (startsWithCI(wowPath, "Spells\\")) {
rest = extractAfterPrefix(wowPath, 7);
return "spell/" + toForwardSlash(rest);
}
// Everything else → misc/{original_path}
return "misc/" + toForwardSlash(wowPath);
}
} // namespace tools
} // namespace wowee

32
tools/asset_extract/path_mapper.hpp Normal file
View file

@ -0,0 +1,32 @@
#pragma once
#include <string>
namespace wowee {
namespace tools {
/**
* Maps WoW virtual paths to reorganized filesystem categories.
*
* Input: WoW virtual path (e.g., "Creature\\Bear\\BearSkin.blp")
* Output: Category-based relative path (e.g., "creature/bear/BearSkin.blp")
*/
class PathMapper {
public:
/**
* Map a WoW virtual path to a reorganized filesystem path.
* @param wowPath Original WoW virtual path (backslash-separated)
* @return Reorganized relative path (forward-slash separated, original casing preserved)
*/
static std::string mapPath(const std::string& wowPath);
private:
// Helpers for prefix matching (case-insensitive)
static bool startsWithCI(const std::string& str, const std::string& prefix);
static std::string toLower(const std::string& str);
static std::string toForwardSlash(const std::string& str);
static std::string extractAfterPrefix(const std::string& path, size_t prefixLen);
};
} // namespace tools
} // namespace wowee

187
tools/blp_convert/main.cpp Normal file
View file

@ -0,0 +1,187 @@
#include "pipeline/blp_loader.hpp"
#define STB_IMAGE_IMPLEMENTATION
#include "stb_image.h"
#define STB_IMAGE_WRITE_IMPLEMENTATION
#include "stb_image_write.h"
#include <algorithm>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
namespace fs = std::filesystem;
using wowee::pipeline::BLPImage;
using wowee::pipeline::BLPLoader;
static std::vector<uint8_t> readFileData(const std::string& path) {
std::ifstream f(path, std::ios::binary | std::ios::ate);
if (!f.is_open()) return {};
auto sz = f.tellg();
if (sz <= 0) return {};
std::vector<uint8_t> data(static_cast<size_t>(sz));
f.seekg(0);
f.read(reinterpret_cast<char*>(data.data()), sz);
return data;
}
static bool convertBlpToPng(const std::string& blpPath) {
auto data = readFileData(blpPath);
if (data.empty()) {
std::cerr << "Failed to read: " << blpPath << "\n";
return false;
}
BLPImage img = BLPLoader::load(data);
if (!img.isValid()) {
std::cerr << "Failed to decode BLP: " << blpPath << "\n";
return false;
}
// Output path: same name with .png
fs::path out = fs::path(blpPath);
out.replace_extension(".png");
if (!stbi_write_png(out.string().c_str(), img.width, img.height, 4,
img.data.data(), img.width * 4)) {
std::cerr << "Failed to write PNG: " << out << "\n";
return false;
}
std::cout << blpPath << " -> " << out.string() << " (" << img.width << "x" << img.height << ")\n";
return true;
}
static bool convertPngToBlp(const std::string& pngPath) {
// Load PNG
int w, h, channels;
unsigned char* pixels = stbi_load(pngPath.c_str(), &w, &h, &channels, 4);
if (!pixels) {
std::cerr << "Failed to read PNG: " << pngPath << "\n";
return false;
}
// Write a simple uncompressed BLP2 (ARGB8888)
fs::path out = fs::path(pngPath);
out.replace_extension(".blp");
std::ofstream f(out.string(), std::ios::binary);
if (!f.is_open()) {
stbi_image_free(pixels);
std::cerr << "Failed to open output: " << out << "\n";
return false;
}
// BLP2 header
f.write("BLP2", 4);
uint32_t version = 1;
f.write(reinterpret_cast<const char*>(&version), 4);
uint8_t compression = 3; // uncompressed
uint8_t alphaDepth = 8;
uint8_t alphaEncoding = 0;
uint8_t hasMips = 0;
f.write(reinterpret_cast<const char*>(&compression), 1);
f.write(reinterpret_cast<const char*>(&alphaDepth), 1);
f.write(reinterpret_cast<const char*>(&alphaEncoding), 1);
f.write(reinterpret_cast<const char*>(&hasMips), 1);
uint32_t width = static_cast<uint32_t>(w);
uint32_t height = static_cast<uint32_t>(h);
f.write(reinterpret_cast<const char*>(&width), 4);
f.write(reinterpret_cast<const char*>(&height), 4);
// Mip offsets (16 entries) — only first used
uint32_t dataSize = width * height * 4;
uint32_t headerSize = 4 + 4 + 4 + 4 + 16 * 4 + 16 * 4 + 256 * 4; // magic+version+dims+mips+palette
uint32_t mipOffsets[16] = {};
uint32_t mipSizes[16] = {};
mipOffsets[0] = headerSize;
mipSizes[0] = dataSize;
f.write(reinterpret_cast<const char*>(mipOffsets), sizeof(mipOffsets));
f.write(reinterpret_cast<const char*>(mipSizes), sizeof(mipSizes));
// Empty palette (256 entries)
uint32_t palette[256] = {};
f.write(reinterpret_cast<const char*>(palette), sizeof(palette));
// Convert RGBA → BGRA for BLP
std::vector<uint8_t> bgra(dataSize);
for (int i = 0; i < w * h; ++i) {
bgra[i * 4 + 0] = pixels[i * 4 + 2]; // B
bgra[i * 4 + 1] = pixels[i * 4 + 1]; // G
bgra[i * 4 + 2] = pixels[i * 4 + 0]; // R
bgra[i * 4 + 3] = pixels[i * 4 + 3]; // A
}
f.write(reinterpret_cast<const char*>(bgra.data()), dataSize);
stbi_image_free(pixels);
std::cout << pngPath << " -> " << out.string() << " (" << w << "x" << h << ")\n";
return true;
}
static void printUsage(const char* prog) {
std::cout << "Usage:\n"
<< " " << prog << " --to-png <file.blp> Convert BLP to PNG\n"
<< " " << prog << " --to-blp <file.png> Convert PNG to BLP\n"
<< " " << prog << " --batch <directory> [--recursive] Batch convert BLP->PNG\n";
}
int main(int argc, char** argv) {
if (argc < 3) {
printUsage(argv[0]);
return 1;
}
std::string mode = argv[1];
if (mode == "--to-png") {
return convertBlpToPng(argv[2]) ? 0 : 1;
}
if (mode == "--to-blp") {
return convertPngToBlp(argv[2]) ? 0 : 1;
}
if (mode == "--batch") {
std::string dir = argv[2];
bool recursive = (argc > 3 && std::strcmp(argv[3], "--recursive") == 0);
int count = 0, failed = 0;
auto processEntry = [&](const fs::directory_entry& entry) {
if (!entry.is_regular_file()) return;
std::string ext = entry.path().extension().string();
std::transform(ext.begin(), ext.end(), ext.begin(),
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
if (ext == ".blp") {
if (convertBlpToPng(entry.path().string())) {
count++;
} else {
failed++;
}
}
};
if (recursive) {
for (const auto& entry : fs::recursive_directory_iterator(dir)) {
processEntry(entry);
}
} else {
for (const auto& entry : fs::directory_iterator(dir)) {
processEntry(entry);
}
}
std::cout << "Batch complete: " << count << " converted, " << failed << " failed\n";
return failed > 0 ? 1 : 0;
}
std::cerr << "Unknown mode: " << mode << "\n";
printUsage(argv[0]);
return 1;
}