mirror of
https://github.com/Kelsidavis/WoWee.git
synced 2026-03-22 23:30:14 +00:00
Add expansion DBC CSVs, Turtle support, and server-specific login
This commit is contained in:
Kelsi
parent
7092844b5e
commit
f247d53309
139 changed files with 676758 additions and 91 deletions
|
|
@ -8,14 +8,19 @@
|
|||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
#include "pipeline/dbc_loader.hpp"
|
||||
|
||||
#ifndef INVALID_HANDLE_VALUE
|
||||
#define INVALID_HANDLE_VALUE ((HANDLE)(long long)-1)
|
||||
#endif
|
||||
|
|
@ -24,6 +29,7 @@ namespace wowee {
|
|||
namespace tools {
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
using wowee::pipeline::DBCFile;
|
||||
|
||||
// Archive descriptor for priority-based loading
|
||||
struct ArchiveDesc {
|
||||
|
|
@ -45,8 +51,237 @@ static std::string normalizeWowPath(const std::string& path) {
|
|||
return n;
|
||||
}
|
||||
|
||||
// Discover archive files in the same priority order as MPQManager
|
||||
static std::vector<ArchiveDesc> discoverArchives(const std::string& mpqDir) {
|
||||
static bool shouldSkipFile(const Extractor::Options& opts, const std::string& wowPath) {
|
||||
if (!opts.skipDbcExtraction) {
|
||||
return false;
|
||||
}
|
||||
std::string n = normalizeWowPath(wowPath);
|
||||
if (n.rfind("dbfilesclient\\", 0) == 0) {
|
||||
if (n.size() >= 4 && n.substr(n.size() - 4) == ".dbc") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static std::vector<uint8_t> readFileBytes(const std::string& path) {
|
||||
std::ifstream f(path, std::ios::binary | std::ios::ate);
|
||||
if (!f) return {};
|
||||
auto size = f.tellg();
|
||||
if (size <= 0) return {};
|
||||
f.seekg(0);
|
||||
std::vector<uint8_t> buf(static_cast<size_t>(size));
|
||||
f.read(reinterpret_cast<char*>(buf.data()), size);
|
||||
return buf;
|
||||
}
|
||||
|
||||
static bool isValidStringOffset(const std::vector<uint8_t>& stringBlock, uint32_t offset) {
|
||||
if (offset >= stringBlock.size()) return false;
|
||||
for (size_t i = offset; i < stringBlock.size(); ++i) {
|
||||
uint8_t c = stringBlock[i];
|
||||
if (c == 0) return true;
|
||||
if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static std::set<uint32_t> detectStringColumns(const DBCFile& dbc,
|
||||
const std::vector<uint8_t>& rawData) {
|
||||
const uint32_t recordCount = dbc.getRecordCount();
|
||||
const uint32_t fieldCount = dbc.getFieldCount();
|
||||
const uint32_t recordSize = dbc.getRecordSize();
|
||||
const uint32_t strBlockSize = dbc.getStringBlockSize();
|
||||
|
||||
constexpr size_t kHeaderSize = 20;
|
||||
const size_t strBlockOffset = kHeaderSize + static_cast<size_t>(recordCount) * recordSize;
|
||||
|
||||
std::vector<uint8_t> stringBlock;
|
||||
if (strBlockSize > 0 && strBlockOffset + strBlockSize <= rawData.size()) {
|
||||
stringBlock.assign(rawData.begin() + strBlockOffset,
|
||||
rawData.begin() + strBlockOffset + strBlockSize);
|
||||
}
|
||||
|
||||
std::set<uint32_t> cols;
|
||||
if (stringBlock.size() <= 1) return cols;
|
||||
|
||||
for (uint32_t col = 0; col < fieldCount; ++col) {
|
||||
bool allZeroOrValid = true;
|
||||
bool hasNonZero = false;
|
||||
|
||||
for (uint32_t row = 0; row < recordCount; ++row) {
|
||||
uint32_t val = dbc.getUInt32(row, col);
|
||||
if (val == 0) continue;
|
||||
hasNonZero = true;
|
||||
if (!isValidStringOffset(stringBlock, val)) {
|
||||
allZeroOrValid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (allZeroOrValid && hasNonZero) {
|
||||
cols.insert(col);
|
||||
}
|
||||
}
|
||||
|
||||
return cols;
|
||||
}
|
||||
|
||||
static std::string csvEscape(const std::string& s) {
|
||||
std::string out;
|
||||
out.reserve(s.size() + 2);
|
||||
out.push_back('"');
|
||||
for (char c : s) {
|
||||
if (c == '"') out.push_back('"');
|
||||
out.push_back(c);
|
||||
}
|
||||
out.push_back('"');
|
||||
return out;
|
||||
}
|
||||
|
||||
static bool convertDbcToCsv(const std::string& dbcPath, const std::string& csvPath) {
|
||||
auto rawData = readFileBytes(dbcPath);
|
||||
if (rawData.size() < 4 || std::memcmp(rawData.data(), "WDBC", 4) != 0) {
|
||||
std::cerr << " DBC missing or not WDBC: " << dbcPath << "\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
DBCFile dbc;
|
||||
if (!dbc.load(rawData) || !dbc.isLoaded()) {
|
||||
std::cerr << " Failed to parse DBC: " << dbcPath << "\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto stringCols = detectStringColumns(dbc, rawData);
|
||||
|
||||
fs::path outPath(csvPath);
|
||||
std::error_code ec;
|
||||
fs::create_directories(outPath.parent_path(), ec);
|
||||
if (ec) {
|
||||
std::cerr << " Failed to create dir: " << outPath.parent_path().string()
|
||||
<< " (" << ec.message() << ")\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
std::ofstream out(csvPath, std::ios::binary);
|
||||
if (!out) {
|
||||
std::cerr << " Failed to write: " << csvPath << "\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
out << "# fields=" << dbc.getFieldCount();
|
||||
if (!stringCols.empty()) {
|
||||
out << " strings=";
|
||||
bool first = true;
|
||||
for (uint32_t col : stringCols) {
|
||||
if (!first) out << ",";
|
||||
out << col;
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
out << "\n";
|
||||
|
||||
for (uint32_t row = 0; row < dbc.getRecordCount(); ++row) {
|
||||
for (uint32_t col = 0; col < dbc.getFieldCount(); ++col) {
|
||||
if (col > 0) out << ",";
|
||||
if (stringCols.count(col)) {
|
||||
out << csvEscape(dbc.getString(row, col));
|
||||
} else {
|
||||
out << dbc.getUInt32(row, col);
|
||||
}
|
||||
}
|
||||
out << "\n";
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static std::vector<std::string> getUsedDbcNamesForExpansion(const std::string& expansion) {
|
||||
// Keep this list small: these are the ~30 tables wowee actually uses.
|
||||
// Other DBCs can remain extracted (ignored) as binary.
|
||||
(void)expansion;
|
||||
return {
|
||||
"AreaTable",
|
||||
"CharSections",
|
||||
"CharHairGeosets",
|
||||
"CharacterFacialHairStyles",
|
||||
"CreatureDisplayInfo",
|
||||
"CreatureDisplayInfoExtra",
|
||||
"CreatureModelData",
|
||||
"Emotes",
|
||||
"EmotesText",
|
||||
"EmotesTextData",
|
||||
"Faction",
|
||||
"FactionTemplate",
|
||||
"GameObjectDisplayInfo",
|
||||
"ItemDisplayInfo",
|
||||
"Light",
|
||||
"LightParams",
|
||||
"LightIntBand",
|
||||
"LightFloatBand",
|
||||
"Map",
|
||||
"SkillLine",
|
||||
"SkillLineAbility",
|
||||
"Spell",
|
||||
"SpellIcon",
|
||||
"Talent",
|
||||
"TalentTab",
|
||||
"TaxiNodes",
|
||||
"TaxiPath",
|
||||
"TaxiPathNode",
|
||||
"TransportAnimation",
|
||||
"WorldMapArea",
|
||||
};
|
||||
}
|
||||
|
||||
static std::unordered_set<std::string> buildWantedDbcSet(const Extractor::Options& opts) {
|
||||
std::unordered_set<std::string> wanted;
|
||||
if (!opts.onlyUsedDbcs) {
|
||||
return wanted;
|
||||
}
|
||||
|
||||
for (const auto& base : getUsedDbcNamesForExpansion(opts.expansion)) {
|
||||
// normalizeWowPath lowercases and uses backslashes.
|
||||
wanted.insert(normalizeWowPath("DBFilesClient\\" + base + ".dbc"));
|
||||
}
|
||||
return wanted;
|
||||
}
|
||||
|
||||
// Known WoW client locales
|
||||
static const std::vector<std::string> kKnownLocales = {
|
||||
"enUS", "enGB", "deDE", "frFR", "esES", "esMX",
|
||||
"ruRU", "koKR", "zhCN", "zhTW", "ptBR"
|
||||
};
|
||||
|
||||
std::string Extractor::detectExpansion(const std::string& mpqDir) {
|
||||
if (fs::exists(mpqDir + "/lichking.MPQ"))
|
||||
return "wotlk";
|
||||
if (fs::exists(mpqDir + "/expansion.MPQ"))
|
||||
return "tbc";
|
||||
// Turtle WoW typically uses vanilla-era base MPQs plus letter patch MPQs (patch-a.mpq ... patch-z.mpq).
|
||||
if (fs::exists(mpqDir + "/dbc.MPQ") || fs::exists(mpqDir + "/terrain.MPQ")) {
|
||||
for (char c = 'a'; c <= 'z'; ++c) {
|
||||
if (fs::exists(mpqDir + std::string("/patch-") + c + ".mpq") ||
|
||||
fs::exists(mpqDir + std::string("/Patch-") + static_cast<char>(std::toupper(c)) + ".mpq")) {
|
||||
return "turtle";
|
||||
}
|
||||
}
|
||||
return "classic";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
std::string Extractor::detectLocale(const std::string& mpqDir) {
|
||||
for (const auto& loc : kKnownLocales) {
|
||||
if (fs::is_directory(mpqDir + "/" + loc))
|
||||
return loc;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
// Discover archive files with expansion-specific and locale-aware loading
|
||||
static std::vector<ArchiveDesc> discoverArchives(const std::string& mpqDir,
|
||||
const std::string& expansion,
|
||||
const std::string& locale) {
|
||||
std::vector<ArchiveDesc> result;
|
||||
|
||||
auto tryAdd = [&](const std::string& name, int prio) {
|
||||
|
|
@ -56,42 +291,120 @@ static std::vector<ArchiveDesc> discoverArchives(const std::string& mpqDir) {
|
|||
}
|
||||
};
|
||||
|
||||
// Base archives (priority 100)
|
||||
tryAdd("common.MPQ", 100);
|
||||
tryAdd("common-2.MPQ", 100);
|
||||
tryAdd("expansion.MPQ", 100);
|
||||
tryAdd("lichking.MPQ", 100);
|
||||
if (expansion == "classic" || expansion == "turtle") {
|
||||
// Vanilla-era base archives (also used by Turtle WoW clients)
|
||||
tryAdd("base.MPQ", 90);
|
||||
tryAdd("base.mpq", 90);
|
||||
tryAdd("backup.MPQ", 95);
|
||||
tryAdd("backup.mpq", 95);
|
||||
tryAdd("dbc.MPQ", 100);
|
||||
tryAdd("dbc.mpq", 100);
|
||||
tryAdd("fonts.MPQ", 100);
|
||||
tryAdd("fonts.mpq", 100);
|
||||
tryAdd("interface.MPQ", 100);
|
||||
tryAdd("interface.mpq", 100);
|
||||
tryAdd("misc.MPQ", 100);
|
||||
tryAdd("misc.mpq", 100);
|
||||
tryAdd("model.MPQ", 100);
|
||||
tryAdd("model.mpq", 100);
|
||||
tryAdd("sound.MPQ", 100);
|
||||
tryAdd("sound.mpq", 100);
|
||||
tryAdd("speech.MPQ", 100);
|
||||
tryAdd("speech.mpq", 100);
|
||||
tryAdd("terrain.MPQ", 100);
|
||||
tryAdd("terrain.mpq", 100);
|
||||
tryAdd("texture.MPQ", 100);
|
||||
tryAdd("texture.mpq", 100);
|
||||
tryAdd("wmo.MPQ", 100);
|
||||
tryAdd("wmo.mpq", 100);
|
||||
|
||||
// Patch archives (priority 150-500)
|
||||
tryAdd("patch.MPQ", 150);
|
||||
tryAdd("patch-2.MPQ", 200);
|
||||
tryAdd("patch-3.MPQ", 300);
|
||||
tryAdd("patch-4.MPQ", 400);
|
||||
tryAdd("patch-5.MPQ", 500);
|
||||
// Patches
|
||||
tryAdd("patch.MPQ", 150);
|
||||
tryAdd("patch.mpq", 150);
|
||||
for (int i = 1; i <= 9; ++i) {
|
||||
tryAdd("patch-" + std::to_string(i) + ".MPQ", 160 + (i * 10));
|
||||
tryAdd("patch-" + std::to_string(i) + ".mpq", 160 + (i * 10));
|
||||
}
|
||||
// Turtle WoW uses letter patch MPQs (patch-a.mpq ... patch-z.mpq).
|
||||
for (char c = 'a'; c <= 'z'; ++c) {
|
||||
tryAdd(std::string("patch-") + c + ".mpq", 800 + (c - 'a'));
|
||||
tryAdd(std::string("Patch-") + static_cast<char>(std::toupper(c)) + ".mpq", 900 + (c - 'a'));
|
||||
}
|
||||
|
||||
// Letter patches (priority 800-925)
|
||||
for (char c = 'a'; c <= 'z'; ++c) {
|
||||
std::string lower = std::string("patch-") + c + ".mpq";
|
||||
std::string upper = std::string("Patch-") + static_cast<char>(std::toupper(c)) + ".mpq";
|
||||
int prioLower = 800 + (c - 'a');
|
||||
int prioUpper = 900 + (c - 'a');
|
||||
tryAdd(lower, prioLower);
|
||||
tryAdd(upper, prioUpper);
|
||||
// Locale
|
||||
if (!locale.empty()) {
|
||||
tryAdd(locale + "/base-" + locale + ".MPQ", 230);
|
||||
tryAdd(locale + "/speech-" + locale + ".MPQ", 240);
|
||||
tryAdd(locale + "/locale-" + locale + ".MPQ", 250);
|
||||
tryAdd(locale + "/patch-" + locale + ".MPQ", 450);
|
||||
}
|
||||
} else if (expansion == "tbc") {
|
||||
// TBC 2.4.x base archives
|
||||
tryAdd("common.MPQ", 100);
|
||||
tryAdd("common-2.MPQ", 100);
|
||||
tryAdd("expansion.MPQ", 100);
|
||||
|
||||
// Patches
|
||||
tryAdd("patch.MPQ", 150);
|
||||
tryAdd("patch-2.MPQ", 200);
|
||||
tryAdd("patch-3.MPQ", 300);
|
||||
tryAdd("patch-4.MPQ", 400);
|
||||
tryAdd("patch-5.MPQ", 500);
|
||||
|
||||
// Letter patches
|
||||
for (char c = 'a'; c <= 'z'; ++c) {
|
||||
tryAdd(std::string("patch-") + c + ".mpq", 800 + (c - 'a'));
|
||||
tryAdd(std::string("Patch-") + static_cast<char>(std::toupper(c)) + ".mpq", 900 + (c - 'a'));
|
||||
}
|
||||
|
||||
// Locale
|
||||
if (!locale.empty()) {
|
||||
tryAdd(locale + "/backup-" + locale + ".MPQ", 225);
|
||||
tryAdd(locale + "/base-" + locale + ".MPQ", 230);
|
||||
tryAdd(locale + "/speech-" + locale + ".MPQ", 240);
|
||||
tryAdd(locale + "/expansion-speech-" + locale + ".MPQ", 245);
|
||||
tryAdd(locale + "/expansion-locale-" + locale + ".MPQ", 246);
|
||||
tryAdd(locale + "/locale-" + locale + ".MPQ", 250);
|
||||
tryAdd(locale + "/patch-" + locale + ".MPQ", 450);
|
||||
tryAdd(locale + "/patch-" + locale + "-2.MPQ", 460);
|
||||
tryAdd(locale + "/patch-" + locale + "-3.MPQ", 470);
|
||||
}
|
||||
} else {
|
||||
// WotLK 3.3.5a (default)
|
||||
tryAdd("common.MPQ", 100);
|
||||
tryAdd("common-2.MPQ", 100);
|
||||
tryAdd("expansion.MPQ", 100);
|
||||
tryAdd("lichking.MPQ", 100);
|
||||
|
||||
// Patches
|
||||
tryAdd("patch.MPQ", 150);
|
||||
tryAdd("patch-2.MPQ", 200);
|
||||
tryAdd("patch-3.MPQ", 300);
|
||||
tryAdd("patch-4.MPQ", 400);
|
||||
tryAdd("patch-5.MPQ", 500);
|
||||
|
||||
// Letter patches
|
||||
for (char c = 'a'; c <= 'z'; ++c) {
|
||||
tryAdd(std::string("patch-") + c + ".mpq", 800 + (c - 'a'));
|
||||
tryAdd(std::string("Patch-") + static_cast<char>(std::toupper(c)) + ".mpq", 900 + (c - 'a'));
|
||||
}
|
||||
|
||||
// Locale
|
||||
if (!locale.empty()) {
|
||||
tryAdd(locale + "/backup-" + locale + ".MPQ", 225);
|
||||
tryAdd(locale + "/base-" + locale + ".MPQ", 230);
|
||||
tryAdd(locale + "/speech-" + locale + ".MPQ", 240);
|
||||
tryAdd(locale + "/expansion-speech-" + locale + ".MPQ", 245);
|
||||
tryAdd(locale + "/expansion-locale-" + locale + ".MPQ", 246);
|
||||
tryAdd(locale + "/lichking-speech-" + locale + ".MPQ", 248);
|
||||
tryAdd(locale + "/lichking-locale-" + locale + ".MPQ", 249);
|
||||
tryAdd(locale + "/locale-" + locale + ".MPQ", 250);
|
||||
tryAdd(locale + "/patch-" + locale + ".MPQ", 450);
|
||||
tryAdd(locale + "/patch-" + locale + "-2.MPQ", 460);
|
||||
tryAdd(locale + "/patch-" + locale + "-3.MPQ", 470);
|
||||
}
|
||||
}
|
||||
|
||||
// Locale archives
|
||||
tryAdd("enUS/backup-enUS.MPQ", 230);
|
||||
tryAdd("enUS/base-enUS.MPQ", 235);
|
||||
tryAdd("enUS/speech-enUS.MPQ", 240);
|
||||
tryAdd("enUS/expansion-speech-enUS.MPQ", 245);
|
||||
tryAdd("enUS/expansion-locale-enUS.MPQ", 246);
|
||||
tryAdd("enUS/lichking-speech-enUS.MPQ", 248);
|
||||
tryAdd("enUS/lichking-locale-enUS.MPQ", 249);
|
||||
tryAdd("enUS/locale-enUS.MPQ", 250);
|
||||
tryAdd("enUS/patch-enUS.MPQ", 450);
|
||||
tryAdd("enUS/patch-enUS-2.MPQ", 460);
|
||||
tryAdd("enUS/patch-enUS-3.MPQ", 470);
|
||||
|
||||
// Sort by priority so highest-priority archives are last
|
||||
// (we'll iterate highest-prio first when extracting)
|
||||
std::sort(result.begin(), result.end(),
|
||||
|
|
@ -104,7 +417,7 @@ bool Extractor::enumerateFiles(const Options& opts,
|
|||
std::vector<std::string>& outFiles) {
|
||||
// Open all archives, enumerate files from highest priority to lowest.
|
||||
// Use a set to deduplicate (highest-priority version wins).
|
||||
auto archives = discoverArchives(opts.mpqDir);
|
||||
auto archives = discoverArchives(opts.mpqDir, opts.expansion, opts.locale);
|
||||
if (archives.empty()) {
|
||||
std::cerr << "No MPQ archives found in: " << opts.mpqDir << "\n";
|
||||
return false;
|
||||
|
|
@ -112,6 +425,8 @@ bool Extractor::enumerateFiles(const Options& opts,
|
|||
|
||||
std::cout << "Found " << archives.size() << " MPQ archives\n";
|
||||
|
||||
const auto wantedDbcs = buildWantedDbcSet(opts);
|
||||
|
||||
// Enumerate from highest priority first so first-seen files win
|
||||
std::set<std::string> seenNormalized;
|
||||
std::vector<std::pair<std::string, std::string>> fileList; // (original name, archive path)
|
||||
|
|
@ -138,7 +453,14 @@ bool Extractor::enumerateFiles(const Options& opts,
|
|||
continue;
|
||||
}
|
||||
|
||||
if (shouldSkipFile(opts, fileName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string norm = normalizeWowPath(fileName);
|
||||
if (opts.onlyUsedDbcs && !wantedDbcs.empty() && !wantedDbcs.contains(norm)) {
|
||||
continue;
|
||||
}
|
||||
if (seenNormalized.insert(norm).second) {
|
||||
// First time seeing this file — this is the highest-priority version
|
||||
outFiles.push_back(fileName);
|
||||
|
|
@ -176,7 +498,7 @@ bool Extractor::run(const Options& opts) {
|
|||
return false;
|
||||
}
|
||||
|
||||
auto archives = discoverArchives(opts.mpqDir);
|
||||
auto archives = discoverArchives(opts.mpqDir, opts.expansion, opts.locale);
|
||||
|
||||
// Determine thread count
|
||||
int numThreads = opts.threads;
|
||||
|
|
@ -371,6 +693,41 @@ bool Extractor::run(const Options& opts) {
|
|||
|
||||
auto elapsed = std::chrono::steady_clock::now() - startTime;
|
||||
auto secs = std::chrono::duration_cast<std::chrono::seconds>(elapsed).count();
|
||||
|
||||
if (opts.generateDbcCsv) {
|
||||
std::cout << "Converting selected DBCs to CSV for committing...\n";
|
||||
const std::string dbcDir = opts.outputDir + "/db";
|
||||
const std::string csvDir = !opts.dbcCsvOutputDir.empty()
|
||||
? opts.dbcCsvOutputDir
|
||||
: (opts.outputDir + "/expansions/" + opts.expansion + "/db");
|
||||
|
||||
uint32_t ok = 0, fail = 0, missing = 0;
|
||||
for (const auto& base : getUsedDbcNamesForExpansion(opts.expansion)) {
|
||||
const std::string in = dbcDir + "/" + base + ".dbc";
|
||||
const std::string out = csvDir + "/" + base + ".csv";
|
||||
if (!fs::exists(in)) {
|
||||
std::cerr << " Missing extracted DBC: " << in << "\n";
|
||||
missing++;
|
||||
continue;
|
||||
}
|
||||
if (!convertDbcToCsv(in, out)) {
|
||||
fail++;
|
||||
} else {
|
||||
ok++;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "DBC CSV conversion: " << ok << " ok";
|
||||
if (missing) std::cout << ", " << missing << " missing";
|
||||
if (fail) std::cout << ", " << fail << " failed";
|
||||
std::cout << "\n";
|
||||
|
||||
if (fail > 0) {
|
||||
std::cerr << "DBC CSV conversion failed for some files\n";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "Done in " << secs / 60 << "m " << secs % 60 << "s\n";
|
||||
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -16,9 +16,15 @@ public:
|
|||
struct Options {
|
||||
std::string mpqDir; // Path to WoW Data directory
|
||||
std::string outputDir; // Output directory for extracted assets
|
||||
std::string expansion; // "classic", "tbc", "wotlk", or "" for auto-detect
|
||||
std::string locale; // "enUS", "deDE", etc., or "" for auto-detect
|
||||
int threads = 0; // 0 = auto-detect
|
||||
bool verify = false; // CRC32 verify after extraction
|
||||
bool verbose = false; // Verbose logging
|
||||
bool generateDbcCsv = false; // Convert selected DBFilesClient/*.dbc to CSV for committing
|
||||
bool skipDbcExtraction = false; // Extract visual assets only (recommended when CSV DBCs are in repo)
|
||||
bool onlyUsedDbcs = false; // Extract only the DBC files wowee uses (implies DBFilesClient/*.dbc filter)
|
||||
std::string dbcCsvOutputDir; // When set, write CSVs into this directory instead of outputDir/expansions/<exp>/db
|
||||
};
|
||||
|
||||
struct Stats {
|
||||
|
|
@ -28,6 +34,18 @@ public:
|
|||
std::atomic<uint64_t> filesFailed{0};
|
||||
};
|
||||
|
||||
/**
|
||||
* Auto-detect expansion from files in mpqDir.
|
||||
* @return "classic", "tbc", "wotlk", or "" if unknown
|
||||
*/
|
||||
static std::string detectExpansion(const std::string& mpqDir);
|
||||
|
||||
/**
|
||||
* Auto-detect locale by scanning for locale subdirectories.
|
||||
* @return locale string like "enUS", or "" if none found
|
||||
*/
|
||||
static std::string detectLocale(const std::string& mpqDir);
|
||||
|
||||
/**
|
||||
* Run the extraction pipeline
|
||||
* @return true on success
|
||||
|
|
|
|||
|
|
@ -13,8 +13,13 @@ static void printUsage(const char* prog) {
|
|||
<< " --output <path> Output directory for extracted assets\n"
|
||||
<< "\n"
|
||||
<< "Options:\n"
|
||||
<< " --expansion <id> Expansion ID (classic/tbc/wotlk/cata).\n"
|
||||
<< " Output goes to <output>/expansions/<id>/\n"
|
||||
<< " --expansion <id> Expansion: classic, turtle, tbc, wotlk (default: auto-detect)\n"
|
||||
<< " --locale <id> Locale: enUS, deDE, frFR, etc. (default: auto-detect)\n"
|
||||
<< " --only-used-dbcs Extract only the DBCs wowee uses (no other assets)\n"
|
||||
<< " --skip-dbc Do not extract DBFilesClient/*.dbc (visual assets only)\n"
|
||||
<< " --dbc-csv Convert selected DBFilesClient/*.dbc to CSV under\n"
|
||||
<< " <output>/expansions/<expansion>/db/*.csv (for committing)\n"
|
||||
<< " --dbc-csv-out <dir> Write CSV DBCs into <dir> (overrides default output path)\n"
|
||||
<< " --verify CRC32 verify all extracted files\n"
|
||||
<< " --threads <N> Number of extraction threads (default: auto)\n"
|
||||
<< " --verbose Verbose output\n"
|
||||
|
|
@ -24,6 +29,7 @@ static void printUsage(const char* prog) {
|
|||
int main(int argc, char** argv) {
|
||||
wowee::tools::Extractor::Options opts;
|
||||
std::string expansion;
|
||||
std::string locale;
|
||||
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
if (std::strcmp(argv[i], "--mpq-dir") == 0 && i + 1 < argc) {
|
||||
|
|
@ -32,8 +38,18 @@ int main(int argc, char** argv) {
|
|||
opts.outputDir = argv[++i];
|
||||
} else if (std::strcmp(argv[i], "--expansion") == 0 && i + 1 < argc) {
|
||||
expansion = argv[++i];
|
||||
} else if (std::strcmp(argv[i], "--locale") == 0 && i + 1 < argc) {
|
||||
locale = argv[++i];
|
||||
} else if (std::strcmp(argv[i], "--threads") == 0 && i + 1 < argc) {
|
||||
opts.threads = std::atoi(argv[++i]);
|
||||
} else if (std::strcmp(argv[i], "--only-used-dbcs") == 0) {
|
||||
opts.onlyUsedDbcs = true;
|
||||
} else if (std::strcmp(argv[i], "--skip-dbc") == 0) {
|
||||
opts.skipDbcExtraction = true;
|
||||
} else if (std::strcmp(argv[i], "--dbc-csv") == 0) {
|
||||
opts.generateDbcCsv = true;
|
||||
} else if (std::strcmp(argv[i], "--dbc-csv-out") == 0 && i + 1 < argc) {
|
||||
opts.dbcCsvOutputDir = argv[++i];
|
||||
} else if (std::strcmp(argv[i], "--verify") == 0) {
|
||||
opts.verify = true;
|
||||
} else if (std::strcmp(argv[i], "--verbose") == 0) {
|
||||
|
|
@ -54,16 +70,48 @@ int main(int argc, char** argv) {
|
|||
return 1;
|
||||
}
|
||||
|
||||
// If --expansion given, redirect output into expansions/<id>/ subdirectory
|
||||
if (!expansion.empty()) {
|
||||
opts.outputDir += "/expansions/" + expansion;
|
||||
// Auto-detect expansion if not specified
|
||||
if (expansion.empty() || expansion == "auto") {
|
||||
expansion = wowee::tools::Extractor::detectExpansion(opts.mpqDir);
|
||||
if (expansion.empty()) {
|
||||
std::cerr << "Error: Could not auto-detect expansion. No known MPQ archives found in: "
|
||||
<< opts.mpqDir << "\n"
|
||||
<< "Specify manually with --expansion classic|tbc|wotlk\n";
|
||||
return 1;
|
||||
}
|
||||
std::cout << "Auto-detected expansion: " << expansion << "\n";
|
||||
}
|
||||
opts.expansion = expansion;
|
||||
|
||||
// Auto-detect locale if not specified
|
||||
if (locale.empty() || locale == "auto") {
|
||||
locale = wowee::tools::Extractor::detectLocale(opts.mpqDir);
|
||||
if (locale.empty()) {
|
||||
std::cerr << "Warning: No locale directory found, skipping locale-specific archives\n";
|
||||
} else {
|
||||
std::cout << "Auto-detected locale: " << locale << "\n";
|
||||
}
|
||||
}
|
||||
opts.locale = locale;
|
||||
|
||||
std::cout << "=== Wowee Asset Extractor ===\n";
|
||||
std::cout << "MPQ directory: " << opts.mpqDir << "\n";
|
||||
std::cout << "Output: " << opts.outputDir << "\n";
|
||||
if (!expansion.empty()) {
|
||||
std::cout << "Expansion: " << expansion << "\n";
|
||||
std::cout << "Expansion: " << expansion << "\n";
|
||||
if (!locale.empty()) {
|
||||
std::cout << "Locale: " << locale << "\n";
|
||||
}
|
||||
if (opts.onlyUsedDbcs) {
|
||||
std::cout << "Mode: only-used-dbcs\n";
|
||||
}
|
||||
if (opts.skipDbcExtraction) {
|
||||
std::cout << "DBC extract: skipped\n";
|
||||
}
|
||||
if (opts.generateDbcCsv) {
|
||||
std::cout << "DBC CSV: enabled\n";
|
||||
if (!opts.dbcCsvOutputDir.empty()) {
|
||||
std::cout << "DBC CSV out: " << opts.dbcCsvOutputDir << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
if (!wowee::tools::Extractor::run(opts)) {
|
||||
|
|
|
|||
196
tools/dbc_to_csv/main.cpp
Normal file
196
tools/dbc_to_csv/main.cpp
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
/**
|
||||
* dbc_to_csv - Convert binary WDBC files to CSV text format.
|
||||
*
|
||||
* Usage: dbc_to_csv <input.dbc> <output.csv>
|
||||
*
|
||||
* Output format:
|
||||
* Line 1: # fields=N strings=I,J,K,... (metadata)
|
||||
* Lines 2+: one record per line, comma-separated fields
|
||||
* String fields are double-quoted with escaped inner quotes.
|
||||
* Numeric fields are plain uint32.
|
||||
*
|
||||
* String column auto-detection:
|
||||
* A column is marked as "string" when every non-zero value in that column
|
||||
* is a valid offset into the WDBC string block (points to a printable,
|
||||
* null-terminated string and doesn't exceed the block size).
|
||||
*/
|
||||
|
||||
#include "pipeline/dbc_loader.hpp"
|
||||
#include "core/logger.hpp"
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
using wowee::pipeline::DBCFile;
|
||||
|
||||
namespace {
|
||||
|
||||
// Read entire file into memory.
|
||||
std::vector<uint8_t> readFileBytes(const std::string& path) {
|
||||
std::ifstream f(path, std::ios::binary | std::ios::ate);
|
||||
if (!f) return {};
|
||||
auto size = f.tellg();
|
||||
if (size <= 0) return {};
|
||||
f.seekg(0);
|
||||
std::vector<uint8_t> buf(static_cast<size_t>(size));
|
||||
f.read(reinterpret_cast<char*>(buf.data()), size);
|
||||
return buf;
|
||||
}
|
||||
|
||||
// Check whether offset points to a plausible string in the string block.
|
||||
bool isValidStringOffset(const std::vector<uint8_t>& stringBlock, uint32_t offset) {
|
||||
if (offset >= stringBlock.size()) return false;
|
||||
// Must be null-terminated within the block and contain only printable/whitespace bytes.
|
||||
for (size_t i = offset; i < stringBlock.size(); ++i) {
|
||||
uint8_t c = stringBlock[i];
|
||||
if (c == 0) return true; // found terminator
|
||||
if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') return false;
|
||||
}
|
||||
return false; // ran off end without terminator
|
||||
}
|
||||
|
||||
// Detect which columns are string columns.
|
||||
std::set<uint32_t> detectStringColumns(const DBCFile& dbc,
|
||||
const std::vector<uint8_t>& rawData) {
|
||||
// Reconstruct the string block from the raw file.
|
||||
// Header is 20 bytes, then recordCount * recordSize bytes of records, then string block.
|
||||
uint32_t recordCount = dbc.getRecordCount();
|
||||
uint32_t fieldCount = dbc.getFieldCount();
|
||||
uint32_t recordSize = dbc.getRecordSize();
|
||||
uint32_t strBlockSize = dbc.getStringBlockSize();
|
||||
|
||||
size_t strBlockOffset = 20 + static_cast<size_t>(recordCount) * recordSize;
|
||||
std::vector<uint8_t> stringBlock;
|
||||
if (strBlockSize > 0 && strBlockOffset + strBlockSize <= rawData.size()) {
|
||||
stringBlock.assign(rawData.begin() + strBlockOffset,
|
||||
rawData.begin() + strBlockOffset + strBlockSize);
|
||||
}
|
||||
|
||||
std::set<uint32_t> stringCols;
|
||||
|
||||
// If no string block (or trivial size), no string columns.
|
||||
if (stringBlock.size() <= 1) return stringCols;
|
||||
|
||||
for (uint32_t col = 0; col < fieldCount; ++col) {
|
||||
bool allZeroOrValid = true;
|
||||
bool hasNonZero = false;
|
||||
|
||||
for (uint32_t row = 0; row < recordCount; ++row) {
|
||||
uint32_t val = dbc.getUInt32(row, col);
|
||||
if (val == 0) continue;
|
||||
hasNonZero = true;
|
||||
if (!isValidStringOffset(stringBlock, val)) {
|
||||
allZeroOrValid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (allZeroOrValid && hasNonZero) {
|
||||
stringCols.insert(col);
|
||||
}
|
||||
}
|
||||
|
||||
return stringCols;
|
||||
}
|
||||
|
||||
// Escape a string for CSV (double-quote, escape inner quotes).
|
||||
std::string csvEscape(const std::string& s) {
|
||||
std::string out;
|
||||
out.reserve(s.size() + 2);
|
||||
out += '"';
|
||||
for (char c : s) {
|
||||
if (c == '"') out += '"'; // double the quote
|
||||
out += c;
|
||||
}
|
||||
out += '"';
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc != 3) {
|
||||
std::cerr << "Usage: dbc_to_csv <input.dbc> <output.csv>\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
const std::string inputPath = argv[1];
|
||||
const std::string outputPath = argv[2];
|
||||
|
||||
// Read input file.
|
||||
auto rawData = readFileBytes(inputPath);
|
||||
if (rawData.empty()) {
|
||||
std::cerr << "Error: cannot read " << inputPath << "\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
// This tool is for converting binary WDBC (.dbc) files only.
|
||||
if (rawData.size() < 4 || std::memcmp(rawData.data(), "WDBC", 4) != 0) {
|
||||
std::cerr << "Error: input is not a binary WDBC DBC file: " << inputPath << "\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Load as WDBC.
|
||||
DBCFile dbc;
|
||||
if (!dbc.load(rawData)) {
|
||||
std::cerr << "Error: failed to parse DBC file " << inputPath << "\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
uint32_t recordCount = dbc.getRecordCount();
|
||||
uint32_t fieldCount = dbc.getFieldCount();
|
||||
|
||||
// Detect string columns.
|
||||
auto stringCols = detectStringColumns(dbc, rawData);
|
||||
|
||||
// Ensure output directory exists.
|
||||
std::filesystem::path outDir = std::filesystem::path(outputPath).parent_path();
|
||||
if (!outDir.empty()) {
|
||||
std::filesystem::create_directories(outDir);
|
||||
}
|
||||
|
||||
// Write CSV.
|
||||
std::ofstream out(outputPath);
|
||||
if (!out) {
|
||||
std::cerr << "Error: cannot write " << outputPath << "\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Metadata line.
|
||||
out << "# fields=" << fieldCount;
|
||||
if (!stringCols.empty()) {
|
||||
out << " strings=";
|
||||
bool first = true;
|
||||
for (uint32_t col : stringCols) {
|
||||
if (!first) out << ',';
|
||||
out << col;
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
out << '\n';
|
||||
|
||||
// Data rows.
|
||||
for (uint32_t row = 0; row < recordCount; ++row) {
|
||||
for (uint32_t col = 0; col < fieldCount; ++col) {
|
||||
if (col > 0) out << ',';
|
||||
if (stringCols.count(col)) {
|
||||
out << csvEscape(dbc.getString(row, col));
|
||||
} else {
|
||||
out << dbc.getUInt32(row, col);
|
||||
}
|
||||
}
|
||||
out << '\n';
|
||||
}
|
||||
|
||||
out.close();
|
||||
|
||||
std::cout << std::filesystem::path(inputPath).filename().string()
|
||||
<< " -> " << std::filesystem::path(outputPath).filename().string()
|
||||
<< " (" << recordCount << " records, " << fieldCount << " fields, "
|
||||
<< stringCols.size() << " string cols)\n";
|
||||
return 0;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue