mirror of
https://github.com/Kelsidavis/WoWee.git
synced 2026-03-24 08:00:14 +00:00
Replace MPQ runtime with loose file asset system
Extract assets from MPQ archives into organized loose files indexed by manifest.json, enabling fully parallel reads without StormLib serialization. Add asset_extract and blp_convert tools, PNG texture override support.
This commit is contained in:
Kelsi
parent
5fda1a3157
commit
aa16a687c2
16 changed files with 1427 additions and 101 deletions
|
|
@ -3,8 +3,11 @@
|
|||
#include "core/memory_monitor.hpp"
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <limits>
|
||||
|
||||
#include "stb_image.h"
|
||||
|
||||
namespace wowee {
|
||||
namespace pipeline {
|
||||
|
||||
|
|
@ -40,26 +43,36 @@ bool AssetManager::initialize(const std::string& dataPath_) {
|
|||
dataPath = dataPath_;
|
||||
LOG_INFO("Initializing asset manager with data path: ", dataPath);
|
||||
|
||||
// Initialize MPQ manager
|
||||
if (!mpqManager.initialize(dataPath)) {
|
||||
LOG_ERROR("Failed to initialize MPQ manager");
|
||||
setupFileCacheBudget();
|
||||
|
||||
std::string manifestPath = dataPath + "/manifest.json";
|
||||
if (!std::filesystem::exists(manifestPath)) {
|
||||
LOG_ERROR("manifest.json not found in: ", dataPath);
|
||||
LOG_ERROR("Run asset_extract to extract MPQ archives first");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Set dynamic file cache budget based on available RAM.
|
||||
// Bias toward MPQ decompressed-file caching to minimize runtime read/decompress stalls.
|
||||
if (!manifest_.load(manifestPath)) {
|
||||
LOG_ERROR("Failed to load manifest");
|
||||
return false;
|
||||
}
|
||||
|
||||
initialized = true;
|
||||
LOG_INFO("Asset manager initialized: ", manifest_.getEntryCount(),
|
||||
" files indexed (file cache: ", fileCacheBudget / (1024 * 1024), " MB)");
|
||||
return true;
|
||||
}
|
||||
|
||||
void AssetManager::setupFileCacheBudget() {
|
||||
auto& memMonitor = core::MemoryMonitor::getInstance();
|
||||
size_t recommendedBudget = memMonitor.getRecommendedCacheBudget();
|
||||
size_t dynamicBudget = (recommendedBudget * 3) / 4; // 75% of global cache budget
|
||||
size_t dynamicBudget = (recommendedBudget * 3) / 4;
|
||||
|
||||
// The MemoryMonitor recommendation is intentionally aggressive; without a cap, large patch MPQs
|
||||
// can cause the decompressed-file cache to balloon into tens/hundreds of GB and push the OS into swap.
|
||||
// Provide env overrides and clamp to a safe default.
|
||||
const size_t envFixedMB = parseEnvSizeMB("WOWEE_FILE_CACHE_MB");
|
||||
const size_t envMaxMB = parseEnvSizeMB("WOWEE_FILE_CACHE_MAX_MB");
|
||||
|
||||
const size_t minBudgetBytes = 256ull * 1024ull * 1024ull; // 256 MB
|
||||
const size_t defaultMaxBudgetBytes = 32768ull * 1024ull * 1024ull; // 32 GB
|
||||
const size_t minBudgetBytes = 256ull * 1024ull * 1024ull;
|
||||
const size_t defaultMaxBudgetBytes = 32768ull * 1024ull * 1024ull;
|
||||
const size_t maxBudgetBytes = (envMaxMB > 0)
|
||||
? (envMaxMB * 1024ull * 1024ull)
|
||||
: defaultMaxBudgetBytes;
|
||||
|
|
@ -74,13 +87,6 @@ bool AssetManager::initialize(const std::string& dataPath_) {
|
|||
} else {
|
||||
fileCacheBudget = std::clamp(dynamicBudget, minBudgetBytes, maxBudgetBytes);
|
||||
}
|
||||
|
||||
initialized = true;
|
||||
LOG_INFO("Asset manager initialized (file cache: ",
|
||||
fileCacheBudget / (1024 * 1024), " MB, recommended=",
|
||||
recommendedBudget / (1024 * 1024), " MB, max=",
|
||||
maxBudgetBytes / (1024 * 1024), " MB)");
|
||||
return true;
|
||||
}
|
||||
|
||||
void AssetManager::shutdown() {
|
||||
|
|
@ -90,7 +96,6 @@ void AssetManager::shutdown() {
|
|||
|
||||
LOG_INFO("Shutting down asset manager");
|
||||
|
||||
// Log cache statistics
|
||||
if (fileCacheHits + fileCacheMisses > 0) {
|
||||
float hitRate = (float)fileCacheHits / (fileCacheHits + fileCacheMisses) * 100.0f;
|
||||
LOG_INFO("File cache stats: ", fileCacheHits, " hits, ", fileCacheMisses, " misses (",
|
||||
|
|
@ -98,8 +103,6 @@ void AssetManager::shutdown() {
|
|||
}
|
||||
|
||||
clearCache();
|
||||
mpqManager.shutdown();
|
||||
|
||||
initialized = false;
|
||||
}
|
||||
|
||||
|
|
@ -109,19 +112,22 @@ BLPImage AssetManager::loadTexture(const std::string& path) {
|
|||
return BLPImage();
|
||||
}
|
||||
|
||||
// Normalize path
|
||||
std::string normalizedPath = normalizePath(path);
|
||||
|
||||
LOG_DEBUG("Loading texture: ", normalizedPath);
|
||||
|
||||
// Route through readFile() so decompressed MPQ bytes participate in file cache.
|
||||
// Check for PNG override
|
||||
BLPImage pngImage = tryLoadPngOverride(normalizedPath);
|
||||
if (pngImage.isValid()) {
|
||||
return pngImage;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> blpData = readFile(normalizedPath);
|
||||
if (blpData.empty()) {
|
||||
LOG_WARNING("Texture not found: ", normalizedPath);
|
||||
return BLPImage();
|
||||
}
|
||||
|
||||
// Load BLP
|
||||
BLPImage image = BLPLoader::load(blpData);
|
||||
if (!image.isValid()) {
|
||||
LOG_ERROR("Failed to load texture: ", normalizedPath);
|
||||
|
|
@ -132,13 +138,47 @@ BLPImage AssetManager::loadTexture(const std::string& path) {
|
|||
return image;
|
||||
}
|
||||
|
||||
BLPImage AssetManager::tryLoadPngOverride(const std::string& normalizedPath) const {
|
||||
if (normalizedPath.size() < 4) return BLPImage();
|
||||
|
||||
std::string ext = normalizedPath.substr(normalizedPath.size() - 4);
|
||||
if (ext != ".blp") return BLPImage();
|
||||
|
||||
std::string fsPath = manifest_.resolveFilesystemPath(normalizedPath);
|
||||
if (fsPath.empty()) return BLPImage();
|
||||
|
||||
// Replace .blp/.BLP extension with .png
|
||||
std::string pngPath = fsPath.substr(0, fsPath.size() - 4) + ".png";
|
||||
if (!LooseFileReader::fileExists(pngPath)) {
|
||||
return BLPImage();
|
||||
}
|
||||
|
||||
int w, h, channels;
|
||||
unsigned char* pixels = stbi_load(pngPath.c_str(), &w, &h, &channels, 4);
|
||||
if (!pixels) {
|
||||
LOG_WARNING("PNG override exists but failed to load: ", pngPath);
|
||||
return BLPImage();
|
||||
}
|
||||
|
||||
BLPImage image;
|
||||
image.width = w;
|
||||
image.height = h;
|
||||
image.channels = 4;
|
||||
image.format = BLPFormat::BLP2;
|
||||
image.compression = BLPCompression::ARGB8888;
|
||||
image.data.assign(pixels, pixels + (w * h * 4));
|
||||
stbi_image_free(pixels);
|
||||
|
||||
LOG_INFO("PNG override loaded: ", pngPath, " (", w, "x", h, ")");
|
||||
return image;
|
||||
}
|
||||
|
||||
std::shared_ptr<DBCFile> AssetManager::loadDBC(const std::string& name) {
|
||||
if (!initialized) {
|
||||
LOG_ERROR("AssetManager not initialized");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Check cache first
|
||||
auto it = dbcCache.find(name);
|
||||
if (it != dbcCache.end()) {
|
||||
LOG_DEBUG("DBC already loaded (cached): ", name);
|
||||
|
|
@ -147,28 +187,20 @@ std::shared_ptr<DBCFile> AssetManager::loadDBC(const std::string& name) {
|
|||
|
||||
LOG_DEBUG("Loading DBC: ", name);
|
||||
|
||||
// Construct DBC path (DBFilesClient directory)
|
||||
std::string dbcPath = "DBFilesClient\\" + name;
|
||||
|
||||
// Read DBC file from MPQ (must hold readMutex — StormLib is not thread-safe)
|
||||
std::vector<uint8_t> dbcData;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(readMutex);
|
||||
dbcData = mpqManager.readFile(dbcPath);
|
||||
}
|
||||
std::vector<uint8_t> dbcData = readFile(dbcPath);
|
||||
if (dbcData.empty()) {
|
||||
LOG_WARNING("DBC not found: ", dbcPath);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Load DBC
|
||||
auto dbc = std::make_shared<DBCFile>();
|
||||
if (!dbc->load(dbcData)) {
|
||||
LOG_ERROR("Failed to load DBC: ", dbcPath);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Cache the DBC
|
||||
dbcCache[name] = dbc;
|
||||
|
||||
LOG_INFO("Loaded DBC: ", name, " (", dbc->getRecordCount(), " records)");
|
||||
|
|
@ -187,57 +219,45 @@ bool AssetManager::fileExists(const std::string& path) const {
|
|||
if (!initialized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lock(readMutex);
|
||||
return mpqManager.fileExists(normalizePath(path));
|
||||
return manifest_.hasEntry(normalizePath(path));
|
||||
}
|
||||
|
||||
std::vector<uint8_t> AssetManager::readFile(const std::string& path) const {
|
||||
if (!initialized) {
|
||||
return std::vector<uint8_t>();
|
||||
return {};
|
||||
}
|
||||
|
||||
std::string normalized = normalizePath(path);
|
||||
|
||||
// Check cache first
|
||||
{
|
||||
std::lock_guard<std::mutex> cacheLock(cacheMutex);
|
||||
// Check cache first
|
||||
auto it = fileCache.find(normalized);
|
||||
if (it != fileCache.end()) {
|
||||
// Cache hit - update access time and return cached data
|
||||
it->second.lastAccessTime = ++fileCacheAccessCounter;
|
||||
fileCacheHits++;
|
||||
return it->second.data;
|
||||
}
|
||||
}
|
||||
|
||||
// Cache miss path: serialize MPQ reads. Before reading, re-check cache while holding
|
||||
// readMutex so only one thread performs decompression per hot path at a time.
|
||||
std::vector<uint8_t> data;
|
||||
{
|
||||
std::lock_guard<std::mutex> readLock(readMutex);
|
||||
{
|
||||
std::lock_guard<std::mutex> cacheLock(cacheMutex);
|
||||
auto it = fileCache.find(normalized);
|
||||
if (it != fileCache.end()) {
|
||||
it->second.lastAccessTime = ++fileCacheAccessCounter;
|
||||
fileCacheHits++;
|
||||
return it->second.data;
|
||||
}
|
||||
fileCacheMisses++;
|
||||
}
|
||||
data = mpqManager.readFile(normalized);
|
||||
// Read from filesystem (fully parallel, no serialization needed)
|
||||
std::string fsPath = manifest_.resolveFilesystemPath(normalized);
|
||||
if (fsPath.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
auto data = LooseFileReader::readFile(fsPath);
|
||||
if (data.empty()) {
|
||||
return data; // File not found
|
||||
LOG_WARNING("Manifest entry exists but file unreadable: ", fsPath);
|
||||
return data;
|
||||
}
|
||||
|
||||
// Add to cache if within budget
|
||||
size_t fileSize = data.size();
|
||||
if (fileSize > 0 && fileSize < fileCacheBudget / 2) { // Don't cache files > 50% of budget (very aggressive)
|
||||
if (fileSize > 0 && fileSize < fileCacheBudget / 2) {
|
||||
std::lock_guard<std::mutex> cacheLock(cacheMutex);
|
||||
// Evict old entries if needed (LRU)
|
||||
while (fileCacheTotalBytes + fileSize > fileCacheBudget && !fileCache.empty()) {
|
||||
// Find least recently used entry
|
||||
auto lru = fileCache.begin();
|
||||
for (auto it = fileCache.begin(); it != fileCache.end(); ++it) {
|
||||
if (it->second.lastAccessTime < lru->second.lastAccessTime) {
|
||||
|
|
@ -248,7 +268,6 @@ std::vector<uint8_t> AssetManager::readFile(const std::string& path) const {
|
|||
fileCache.erase(lru);
|
||||
}
|
||||
|
||||
// Add new entry
|
||||
CachedFile cached;
|
||||
cached.data = data;
|
||||
cached.lastAccessTime = ++fileCacheAccessCounter;
|
||||
|
|
@ -261,18 +280,16 @@ std::vector<uint8_t> AssetManager::readFile(const std::string& path) const {
|
|||
|
||||
std::vector<uint8_t> AssetManager::readFileOptional(const std::string& path) const {
|
||||
if (!initialized) {
|
||||
return std::vector<uint8_t>();
|
||||
return {};
|
||||
}
|
||||
|
||||
// Avoid MPQManager missing-file warnings for expected probe misses.
|
||||
if (!fileExists(path)) {
|
||||
return std::vector<uint8_t>();
|
||||
return {};
|
||||
}
|
||||
return readFile(path);
|
||||
}
|
||||
|
||||
void AssetManager::clearCache() {
|
||||
std::scoped_lock lock(readMutex, cacheMutex);
|
||||
std::lock_guard<std::mutex> lock(cacheMutex);
|
||||
dbcCache.clear();
|
||||
fileCache.clear();
|
||||
fileCacheTotalBytes = 0;
|
||||
|
|
@ -282,13 +299,9 @@ void AssetManager::clearCache() {
|
|||
|
||||
std::string AssetManager::normalizePath(const std::string& path) const {
|
||||
std::string normalized = path;
|
||||
|
||||
// Convert forward slashes to backslashes (WoW uses backslashes)
|
||||
std::replace(normalized.begin(), normalized.end(), '/', '\\');
|
||||
// Lowercase for case-insensitive cache keys (improves hit rate across mixed-case callers).
|
||||
std::transform(normalized.begin(), normalized.end(), normalized.begin(),
|
||||
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
|
|
|
|||
97
src/pipeline/asset_manifest.cpp
Normal file
97
src/pipeline/asset_manifest.cpp
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
#include "pipeline/asset_manifest.hpp"
|
||||
#include "core/logger.hpp"
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <fstream>
|
||||
#include <filesystem>
|
||||
#include <chrono>
|
||||
|
||||
namespace wowee {
|
||||
namespace pipeline {
|
||||
|
||||
bool AssetManifest::load(const std::string& manifestPath) {
|
||||
auto startTime = std::chrono::steady_clock::now();
|
||||
|
||||
std::ifstream file(manifestPath);
|
||||
if (!file.is_open()) {
|
||||
LOG_ERROR("Failed to open manifest: ", manifestPath);
|
||||
return false;
|
||||
}
|
||||
|
||||
nlohmann::json doc;
|
||||
try {
|
||||
doc = nlohmann::json::parse(file);
|
||||
} catch (const nlohmann::json::parse_error& e) {
|
||||
LOG_ERROR("Failed to parse manifest JSON: ", e.what());
|
||||
return false;
|
||||
}
|
||||
|
||||
// Read header
|
||||
int version = doc.value("version", 0);
|
||||
if (version != 1) {
|
||||
LOG_ERROR("Unsupported manifest version: ", version);
|
||||
return false;
|
||||
}
|
||||
|
||||
basePath_ = doc.value("basePath", "assets");
|
||||
manifestDir_ = std::filesystem::path(manifestPath).parent_path().string();
|
||||
|
||||
// If basePath is relative, resolve against manifest directory
|
||||
if (!basePath_.empty() && basePath_[0] != '/') {
|
||||
basePath_ = manifestDir_ + "/" + basePath_;
|
||||
}
|
||||
|
||||
// Parse entries
|
||||
auto& entriesObj = doc["entries"];
|
||||
if (!entriesObj.is_object()) {
|
||||
LOG_ERROR("Manifest missing 'entries' object");
|
||||
return false;
|
||||
}
|
||||
|
||||
entries_.reserve(entriesObj.size());
|
||||
for (auto& [key, val] : entriesObj.items()) {
|
||||
Entry entry;
|
||||
entry.filesystemPath = val.value("p", "");
|
||||
entry.size = val.value("s", uint64_t(0));
|
||||
|
||||
// Parse CRC32 from hex string
|
||||
std::string hexHash = val.value("h", "");
|
||||
if (!hexHash.empty()) {
|
||||
entry.crc32 = static_cast<uint32_t>(std::strtoul(hexHash.c_str(), nullptr, 16));
|
||||
} else {
|
||||
entry.crc32 = 0;
|
||||
}
|
||||
|
||||
entries_[key] = std::move(entry);
|
||||
}
|
||||
|
||||
loaded_ = true;
|
||||
|
||||
auto elapsed = std::chrono::steady_clock::now() - startTime;
|
||||
auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count();
|
||||
LOG_INFO("Loaded asset manifest: ", entries_.size(), " entries in ", ms, "ms (base: ", basePath_, ")");
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
const AssetManifest::Entry* AssetManifest::lookup(const std::string& normalizedWowPath) const {
|
||||
auto it = entries_.find(normalizedWowPath);
|
||||
if (it != entries_.end()) {
|
||||
return &it->second;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::string AssetManifest::resolveFilesystemPath(const std::string& normalizedWowPath) const {
|
||||
auto it = entries_.find(normalizedWowPath);
|
||||
if (it == entries_.end()) {
|
||||
return {};
|
||||
}
|
||||
return basePath_ + "/" + it->second.filesystemPath;
|
||||
}
|
||||
|
||||
bool AssetManifest::hasEntry(const std::string& normalizedWowPath) const {
|
||||
return entries_.find(normalizedWowPath) != entries_.end();
|
||||
}
|
||||
|
||||
} // namespace pipeline
|
||||
} // namespace wowee
|
||||
47
src/pipeline/loose_file_reader.cpp
Normal file
47
src/pipeline/loose_file_reader.cpp
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
#include "pipeline/loose_file_reader.hpp"
|
||||
#include "core/logger.hpp"
|
||||
#include <fstream>
|
||||
#include <filesystem>
|
||||
|
||||
namespace wowee {
|
||||
namespace pipeline {
|
||||
|
||||
std::vector<uint8_t> LooseFileReader::readFile(const std::string& filesystemPath) {
|
||||
std::ifstream file(filesystemPath, std::ios::binary | std::ios::ate);
|
||||
if (!file.is_open()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
auto size = file.tellg();
|
||||
if (size <= 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<uint8_t> data(static_cast<size_t>(size));
|
||||
file.seekg(0, std::ios::beg);
|
||||
file.read(reinterpret_cast<char*>(data.data()), size);
|
||||
|
||||
if (!file.good()) {
|
||||
LOG_WARNING("Incomplete read of: ", filesystemPath);
|
||||
data.resize(static_cast<size_t>(file.gcount()));
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
bool LooseFileReader::fileExists(const std::string& filesystemPath) {
|
||||
std::error_code ec;
|
||||
return std::filesystem::exists(filesystemPath, ec);
|
||||
}
|
||||
|
||||
uint64_t LooseFileReader::getFileSize(const std::string& filesystemPath) {
|
||||
std::error_code ec;
|
||||
auto size = std::filesystem::file_size(filesystemPath, ec);
|
||||
if (ec) {
|
||||
return 0;
|
||||
}
|
||||
return static_cast<uint64_t>(size);
|
||||
}
|
||||
|
||||
} // namespace pipeline
|
||||
} // namespace wowee
|
||||
|
|
@ -267,7 +267,7 @@ void Minimap::parseTRS() {
|
|||
if (trsParsed || !assetManager) return;
|
||||
trsParsed = true;
|
||||
|
||||
auto data = assetManager->getMPQManager().readFile("Textures\\Minimap\\md5translate.trs");
|
||||
auto data = assetManager->readFile("Textures\\Minimap\\md5translate.trs");
|
||||
if (data.empty()) {
|
||||
LOG_WARNING("Failed to load md5translate.trs");
|
||||
return;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue