mirror of
https://github.com/Kelsidavis/WoWee.git
synced 2026-03-23 07:40:14 +00:00
Replace MPQ runtime with loose file asset system
Extract assets from MPQ archives into organized loose files indexed by manifest.json, enabling fully parallel reads without StormLib serialization. Add asset_extract and blp_convert tools, PNG texture override support.
This commit is contained in:
Kelsi
parent
5fda1a3157
commit
aa16a687c2
16 changed files with 1427 additions and 101 deletions
|
|
@ -3,8 +3,11 @@
|
|||
#include "core/memory_monitor.hpp"
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <limits>
|
||||
|
||||
#include "stb_image.h"
|
||||
|
||||
namespace wowee {
|
||||
namespace pipeline {
|
||||
|
||||
|
|
@ -40,26 +43,36 @@ bool AssetManager::initialize(const std::string& dataPath_) {
|
|||
dataPath = dataPath_;
|
||||
LOG_INFO("Initializing asset manager with data path: ", dataPath);
|
||||
|
||||
// Initialize MPQ manager
|
||||
if (!mpqManager.initialize(dataPath)) {
|
||||
LOG_ERROR("Failed to initialize MPQ manager");
|
||||
setupFileCacheBudget();
|
||||
|
||||
std::string manifestPath = dataPath + "/manifest.json";
|
||||
if (!std::filesystem::exists(manifestPath)) {
|
||||
LOG_ERROR("manifest.json not found in: ", dataPath);
|
||||
LOG_ERROR("Run asset_extract to extract MPQ archives first");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Set dynamic file cache budget based on available RAM.
|
||||
// Bias toward MPQ decompressed-file caching to minimize runtime read/decompress stalls.
|
||||
if (!manifest_.load(manifestPath)) {
|
||||
LOG_ERROR("Failed to load manifest");
|
||||
return false;
|
||||
}
|
||||
|
||||
initialized = true;
|
||||
LOG_INFO("Asset manager initialized: ", manifest_.getEntryCount(),
|
||||
" files indexed (file cache: ", fileCacheBudget / (1024 * 1024), " MB)");
|
||||
return true;
|
||||
}
|
||||
|
||||
void AssetManager::setupFileCacheBudget() {
|
||||
auto& memMonitor = core::MemoryMonitor::getInstance();
|
||||
size_t recommendedBudget = memMonitor.getRecommendedCacheBudget();
|
||||
size_t dynamicBudget = (recommendedBudget * 3) / 4; // 75% of global cache budget
|
||||
size_t dynamicBudget = (recommendedBudget * 3) / 4;
|
||||
|
||||
// The MemoryMonitor recommendation is intentionally aggressive; without a cap, large patch MPQs
|
||||
// can cause the decompressed-file cache to balloon into tens/hundreds of GB and push the OS into swap.
|
||||
// Provide env overrides and clamp to a safe default.
|
||||
const size_t envFixedMB = parseEnvSizeMB("WOWEE_FILE_CACHE_MB");
|
||||
const size_t envMaxMB = parseEnvSizeMB("WOWEE_FILE_CACHE_MAX_MB");
|
||||
|
||||
const size_t minBudgetBytes = 256ull * 1024ull * 1024ull; // 256 MB
|
||||
const size_t defaultMaxBudgetBytes = 32768ull * 1024ull * 1024ull; // 32 GB
|
||||
const size_t minBudgetBytes = 256ull * 1024ull * 1024ull;
|
||||
const size_t defaultMaxBudgetBytes = 32768ull * 1024ull * 1024ull;
|
||||
const size_t maxBudgetBytes = (envMaxMB > 0)
|
||||
? (envMaxMB * 1024ull * 1024ull)
|
||||
: defaultMaxBudgetBytes;
|
||||
|
|
@ -74,13 +87,6 @@ bool AssetManager::initialize(const std::string& dataPath_) {
|
|||
} else {
|
||||
fileCacheBudget = std::clamp(dynamicBudget, minBudgetBytes, maxBudgetBytes);
|
||||
}
|
||||
|
||||
initialized = true;
|
||||
LOG_INFO("Asset manager initialized (file cache: ",
|
||||
fileCacheBudget / (1024 * 1024), " MB, recommended=",
|
||||
recommendedBudget / (1024 * 1024), " MB, max=",
|
||||
maxBudgetBytes / (1024 * 1024), " MB)");
|
||||
return true;
|
||||
}
|
||||
|
||||
void AssetManager::shutdown() {
|
||||
|
|
@ -90,7 +96,6 @@ void AssetManager::shutdown() {
|
|||
|
||||
LOG_INFO("Shutting down asset manager");
|
||||
|
||||
// Log cache statistics
|
||||
if (fileCacheHits + fileCacheMisses > 0) {
|
||||
float hitRate = (float)fileCacheHits / (fileCacheHits + fileCacheMisses) * 100.0f;
|
||||
LOG_INFO("File cache stats: ", fileCacheHits, " hits, ", fileCacheMisses, " misses (",
|
||||
|
|
@ -98,8 +103,6 @@ void AssetManager::shutdown() {
|
|||
}
|
||||
|
||||
clearCache();
|
||||
mpqManager.shutdown();
|
||||
|
||||
initialized = false;
|
||||
}
|
||||
|
||||
|
|
@ -109,19 +112,22 @@ BLPImage AssetManager::loadTexture(const std::string& path) {
|
|||
return BLPImage();
|
||||
}
|
||||
|
||||
// Normalize path
|
||||
std::string normalizedPath = normalizePath(path);
|
||||
|
||||
LOG_DEBUG("Loading texture: ", normalizedPath);
|
||||
|
||||
// Route through readFile() so decompressed MPQ bytes participate in file cache.
|
||||
// Check for PNG override
|
||||
BLPImage pngImage = tryLoadPngOverride(normalizedPath);
|
||||
if (pngImage.isValid()) {
|
||||
return pngImage;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> blpData = readFile(normalizedPath);
|
||||
if (blpData.empty()) {
|
||||
LOG_WARNING("Texture not found: ", normalizedPath);
|
||||
return BLPImage();
|
||||
}
|
||||
|
||||
// Load BLP
|
||||
BLPImage image = BLPLoader::load(blpData);
|
||||
if (!image.isValid()) {
|
||||
LOG_ERROR("Failed to load texture: ", normalizedPath);
|
||||
|
|
@ -132,13 +138,47 @@ BLPImage AssetManager::loadTexture(const std::string& path) {
|
|||
return image;
|
||||
}
|
||||
|
||||
BLPImage AssetManager::tryLoadPngOverride(const std::string& normalizedPath) const {
|
||||
if (normalizedPath.size() < 4) return BLPImage();
|
||||
|
||||
std::string ext = normalizedPath.substr(normalizedPath.size() - 4);
|
||||
if (ext != ".blp") return BLPImage();
|
||||
|
||||
std::string fsPath = manifest_.resolveFilesystemPath(normalizedPath);
|
||||
if (fsPath.empty()) return BLPImage();
|
||||
|
||||
// Replace .blp/.BLP extension with .png
|
||||
std::string pngPath = fsPath.substr(0, fsPath.size() - 4) + ".png";
|
||||
if (!LooseFileReader::fileExists(pngPath)) {
|
||||
return BLPImage();
|
||||
}
|
||||
|
||||
int w, h, channels;
|
||||
unsigned char* pixels = stbi_load(pngPath.c_str(), &w, &h, &channels, 4);
|
||||
if (!pixels) {
|
||||
LOG_WARNING("PNG override exists but failed to load: ", pngPath);
|
||||
return BLPImage();
|
||||
}
|
||||
|
||||
BLPImage image;
|
||||
image.width = w;
|
||||
image.height = h;
|
||||
image.channels = 4;
|
||||
image.format = BLPFormat::BLP2;
|
||||
image.compression = BLPCompression::ARGB8888;
|
||||
image.data.assign(pixels, pixels + (w * h * 4));
|
||||
stbi_image_free(pixels);
|
||||
|
||||
LOG_INFO("PNG override loaded: ", pngPath, " (", w, "x", h, ")");
|
||||
return image;
|
||||
}
|
||||
|
||||
std::shared_ptr<DBCFile> AssetManager::loadDBC(const std::string& name) {
|
||||
if (!initialized) {
|
||||
LOG_ERROR("AssetManager not initialized");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Check cache first
|
||||
auto it = dbcCache.find(name);
|
||||
if (it != dbcCache.end()) {
|
||||
LOG_DEBUG("DBC already loaded (cached): ", name);
|
||||
|
|
@ -147,28 +187,20 @@ std::shared_ptr<DBCFile> AssetManager::loadDBC(const std::string& name) {
|
|||
|
||||
LOG_DEBUG("Loading DBC: ", name);
|
||||
|
||||
// Construct DBC path (DBFilesClient directory)
|
||||
std::string dbcPath = "DBFilesClient\\" + name;
|
||||
|
||||
// Read DBC file from MPQ (must hold readMutex — StormLib is not thread-safe)
|
||||
std::vector<uint8_t> dbcData;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(readMutex);
|
||||
dbcData = mpqManager.readFile(dbcPath);
|
||||
}
|
||||
std::vector<uint8_t> dbcData = readFile(dbcPath);
|
||||
if (dbcData.empty()) {
|
||||
LOG_WARNING("DBC not found: ", dbcPath);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Load DBC
|
||||
auto dbc = std::make_shared<DBCFile>();
|
||||
if (!dbc->load(dbcData)) {
|
||||
LOG_ERROR("Failed to load DBC: ", dbcPath);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Cache the DBC
|
||||
dbcCache[name] = dbc;
|
||||
|
||||
LOG_INFO("Loaded DBC: ", name, " (", dbc->getRecordCount(), " records)");
|
||||
|
|
@ -187,57 +219,45 @@ bool AssetManager::fileExists(const std::string& path) const {
|
|||
if (!initialized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lock(readMutex);
|
||||
return mpqManager.fileExists(normalizePath(path));
|
||||
return manifest_.hasEntry(normalizePath(path));
|
||||
}
|
||||
|
||||
std::vector<uint8_t> AssetManager::readFile(const std::string& path) const {
|
||||
if (!initialized) {
|
||||
return std::vector<uint8_t>();
|
||||
return {};
|
||||
}
|
||||
|
||||
std::string normalized = normalizePath(path);
|
||||
|
||||
// Check cache first
|
||||
{
|
||||
std::lock_guard<std::mutex> cacheLock(cacheMutex);
|
||||
// Check cache first
|
||||
auto it = fileCache.find(normalized);
|
||||
if (it != fileCache.end()) {
|
||||
// Cache hit - update access time and return cached data
|
||||
it->second.lastAccessTime = ++fileCacheAccessCounter;
|
||||
fileCacheHits++;
|
||||
return it->second.data;
|
||||
}
|
||||
}
|
||||
|
||||
// Cache miss path: serialize MPQ reads. Before reading, re-check cache while holding
|
||||
// readMutex so only one thread performs decompression per hot path at a time.
|
||||
std::vector<uint8_t> data;
|
||||
{
|
||||
std::lock_guard<std::mutex> readLock(readMutex);
|
||||
{
|
||||
std::lock_guard<std::mutex> cacheLock(cacheMutex);
|
||||
auto it = fileCache.find(normalized);
|
||||
if (it != fileCache.end()) {
|
||||
it->second.lastAccessTime = ++fileCacheAccessCounter;
|
||||
fileCacheHits++;
|
||||
return it->second.data;
|
||||
}
|
||||
fileCacheMisses++;
|
||||
}
|
||||
data = mpqManager.readFile(normalized);
|
||||
// Read from filesystem (fully parallel, no serialization needed)
|
||||
std::string fsPath = manifest_.resolveFilesystemPath(normalized);
|
||||
if (fsPath.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
auto data = LooseFileReader::readFile(fsPath);
|
||||
if (data.empty()) {
|
||||
return data; // File not found
|
||||
LOG_WARNING("Manifest entry exists but file unreadable: ", fsPath);
|
||||
return data;
|
||||
}
|
||||
|
||||
// Add to cache if within budget
|
||||
size_t fileSize = data.size();
|
||||
if (fileSize > 0 && fileSize < fileCacheBudget / 2) { // Don't cache files > 50% of budget (very aggressive)
|
||||
if (fileSize > 0 && fileSize < fileCacheBudget / 2) {
|
||||
std::lock_guard<std::mutex> cacheLock(cacheMutex);
|
||||
// Evict old entries if needed (LRU)
|
||||
while (fileCacheTotalBytes + fileSize > fileCacheBudget && !fileCache.empty()) {
|
||||
// Find least recently used entry
|
||||
auto lru = fileCache.begin();
|
||||
for (auto it = fileCache.begin(); it != fileCache.end(); ++it) {
|
||||
if (it->second.lastAccessTime < lru->second.lastAccessTime) {
|
||||
|
|
@ -248,7 +268,6 @@ std::vector<uint8_t> AssetManager::readFile(const std::string& path) const {
|
|||
fileCache.erase(lru);
|
||||
}
|
||||
|
||||
// Add new entry
|
||||
CachedFile cached;
|
||||
cached.data = data;
|
||||
cached.lastAccessTime = ++fileCacheAccessCounter;
|
||||
|
|
@ -261,18 +280,16 @@ std::vector<uint8_t> AssetManager::readFile(const std::string& path) const {
|
|||
|
||||
std::vector<uint8_t> AssetManager::readFileOptional(const std::string& path) const {
|
||||
if (!initialized) {
|
||||
return std::vector<uint8_t>();
|
||||
return {};
|
||||
}
|
||||
|
||||
// Avoid MPQManager missing-file warnings for expected probe misses.
|
||||
if (!fileExists(path)) {
|
||||
return std::vector<uint8_t>();
|
||||
return {};
|
||||
}
|
||||
return readFile(path);
|
||||
}
|
||||
|
||||
void AssetManager::clearCache() {
|
||||
std::scoped_lock lock(readMutex, cacheMutex);
|
||||
std::lock_guard<std::mutex> lock(cacheMutex);
|
||||
dbcCache.clear();
|
||||
fileCache.clear();
|
||||
fileCacheTotalBytes = 0;
|
||||
|
|
@ -282,13 +299,9 @@ void AssetManager::clearCache() {
|
|||
|
||||
std::string AssetManager::normalizePath(const std::string& path) const {
|
||||
std::string normalized = path;
|
||||
|
||||
// Convert forward slashes to backslashes (WoW uses backslashes)
|
||||
std::replace(normalized.begin(), normalized.end(), '/', '\\');
|
||||
// Lowercase for case-insensitive cache keys (improves hit rate across mixed-case callers).
|
||||
std::transform(normalized.begin(), normalized.end(), normalized.begin(),
|
||||
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue