diff --git a/include/pipeline/dbc_loader.hpp b/include/pipeline/dbc_loader.hpp index 1651e5d2..148691ae 100644 --- a/include/pipeline/dbc_loader.hpp +++ b/include/pipeline/dbc_loader.hpp @@ -136,6 +136,14 @@ private: * Rebuilds the same in-memory layout as binary load. */ bool loadCSV(const std::vector& csvData); + + /** + * Load from plain CSV format with column-name header row. + * e.g. "id,raceId,sexId,...\n21,1,0,...\n" + * All fields are treated as numeric (uint32 / int32). + */ + bool loadPlainCSV(const std::vector& csvData); + }; } // namespace pipeline diff --git a/src/pipeline/dbc_layout.cpp b/src/pipeline/dbc_layout.cpp index 08730536..7dc1f1af 100644 --- a/src/pipeline/dbc_layout.cpp +++ b/src/pipeline/dbc_layout.cpp @@ -11,6 +11,139 @@ static const DBCLayout* g_activeDBCLayout = nullptr; void setActiveDBCLayout(const DBCLayout* layout) { g_activeDBCLayout = layout; } const DBCLayout* getActiveDBCLayout() { return g_activeDBCLayout; } +void DBCLayout::loadWotlkDefaults() { + layouts_.clear(); + + // Spell.dbc + layouts_["Spell"] = {{{ "ID", 0 }, { "Attributes", 4 }, { "IconID", 133 }, + { "Name", 136 }, { "Tooltip", 139 }, { "Rank", 153 }}}; + + // ItemDisplayInfo.dbc + layouts_["ItemDisplayInfo"] = {{{ "ID", 0 }, { "LeftModel", 1 }, { "LeftModelTexture", 3 }, + { "InventoryIcon", 5 }, { "GeosetGroup1", 7 }, { "GeosetGroup3", 9 }}}; + + // CharSections.dbc + // Binary layout: ID(0) Race(1) Sex(2) Section(3) Variation(4) Color(5) Tex1(6) Tex2(7) Tex3(8) Flags(9) + layouts_["CharSections"] = {{{ "RaceID", 1 }, { "SexID", 2 }, { "BaseSection", 3 }, + { "VariationIndex", 4 }, { "ColorIndex", 5 }, + { "Texture1", 6 }, { "Texture2", 7 }, { "Texture3", 8 }, + { "Flags", 9 }}}; + + // SpellIcon.dbc (Icon.dbc in code but actually SpellIcon) + layouts_["SpellIcon"] = {{{ "ID", 0 }, { "Path", 1 }}}; + + // FactionTemplate.dbc + layouts_["FactionTemplate"] = {{{ "ID", 0 }, { "Faction", 1 }, { "FactionGroup", 3 }, + { "FriendGroup", 4 }, { "EnemyGroup", 5 }, + { "Enemy0", 6 }, { "Enemy1", 7 }, { "Enemy2", 8 }, { "Enemy3", 9 }}}; + + // Faction.dbc + layouts_["Faction"] = {{{ "ID", 0 }, { "ReputationRaceMask0", 2 }, { "ReputationRaceMask1", 3 }, + { "ReputationRaceMask2", 4 }, { "ReputationRaceMask3", 5 }, + { "ReputationBase0", 10 }, { "ReputationBase1", 11 }, + { "ReputationBase2", 12 }, { "ReputationBase3", 13 }}}; + + // AreaTable.dbc + layouts_["AreaTable"] = {{{ "ID", 0 }, { "ExploreFlag", 3 }}}; + + // CreatureDisplayInfoExtra.dbc + layouts_["CreatureDisplayInfoExtra"] = {{{ "ID", 0 }, { "RaceID", 1 }, { "SexID", 2 }, + { "SkinID", 3 }, { "FaceID", 4 }, { "HairStyleID", 5 }, { "HairColorID", 6 }, + { "FacialHairID", 7 }, { "EquipDisplay0", 8 }, { "EquipDisplay1", 9 }, + { "EquipDisplay2", 10 }, { "EquipDisplay3", 11 }, { "EquipDisplay4", 12 }, + { "EquipDisplay5", 13 }, { "EquipDisplay6", 14 }, { "EquipDisplay7", 15 }, + { "EquipDisplay8", 16 }, { "EquipDisplay9", 17 }, { "EquipDisplay10", 18 }, + { "BakeName", 20 }}}; + + // CreatureDisplayInfo.dbc + layouts_["CreatureDisplayInfo"] = {{{ "ID", 0 }, { "ModelID", 1 }, { "ExtraDisplayId", 3 }, + { "Skin1", 6 }, { "Skin2", 7 }, { "Skin3", 8 }}}; + + // TaxiNodes.dbc + layouts_["TaxiNodes"] = {{{ "ID", 0 }, { "MapID", 1 }, { "X", 2 }, { "Y", 3 }, { "Z", 4 }, + { "Name", 5 }, { "MountDisplayIdAllianceFallback", 20 }, + { "MountDisplayIdHordeFallback", 21 }, + { "MountDisplayIdAlliance", 22 }, { "MountDisplayIdHorde", 23 }}}; + + // TaxiPath.dbc + layouts_["TaxiPath"] = {{{ "ID", 0 }, { "FromNode", 1 }, { "ToNode", 2 }, { "Cost", 3 }}}; + + // TaxiPathNode.dbc + layouts_["TaxiPathNode"] = {{{ "ID", 0 }, { "PathID", 1 }, { "NodeIndex", 2 }, + { "MapID", 3 }, { "X", 4 }, { "Y", 5 }, { "Z", 6 }}}; + + // TalentTab.dbc + layouts_["TalentTab"] = {{{ "ID", 0 }, { "Name", 1 }, { "ClassMask", 20 }, + { "OrderIndex", 22 }, { "BackgroundFile", 23 }}}; + + // Talent.dbc + layouts_["Talent"] = {{{ "ID", 0 }, { "TabID", 1 }, { "Row", 2 }, { "Column", 3 }, + { "RankSpell0", 4 }, { "PrereqTalent0", 9 }, { "PrereqRank0", 12 }}}; + + // SkillLineAbility.dbc + layouts_["SkillLineAbility"] = {{{ "SkillLineID", 1 }, { "SpellID", 2 }}}; + + // SkillLine.dbc + layouts_["SkillLine"] = {{{ "ID", 0 }, { "Category", 1 }, { "Name", 3 }}}; + + // Map.dbc + layouts_["Map"] = {{{ "ID", 0 }, { "InternalName", 1 }}}; + + // CreatureModelData.dbc + layouts_["CreatureModelData"] = {{{ "ID", 0 }, { "ModelPath", 2 }}}; + + // CharHairGeosets.dbc + layouts_["CharHairGeosets"] = {{{ "RaceID", 1 }, { "SexID", 2 }, + { "Variation", 3 }, { "GeosetID", 4 }}}; + + // CharacterFacialHairStyles.dbc + layouts_["CharacterFacialHairStyles"] = {{{ "RaceID", 0 }, { "SexID", 1 }, + { "Variation", 2 }, { "Geoset100", 3 }, { "Geoset300", 4 }, { "Geoset200", 5 }}}; + + // GameObjectDisplayInfo.dbc + layouts_["GameObjectDisplayInfo"] = {{{ "ID", 0 }, { "ModelName", 1 }}}; + + // Emotes.dbc + layouts_["Emotes"] = {{{ "ID", 0 }, { "AnimID", 2 }}}; + + // EmotesText.dbc + // Fields 3-18 are 16 EmotesTextData refs: [others+target, target+target, sender+target, ?, + // others+notarget, ?, sender+notarget, ?, female variants...] + layouts_["EmotesText"] = {{{ "ID", 0 }, { "Command", 1 }, { "EmoteRef", 2 }, + { "OthersTargetTextID", 3 }, { "SenderTargetTextID", 5 }, + { "OthersNoTargetTextID", 7 }, { "SenderNoTargetTextID", 9 }}}; + + // EmotesTextData.dbc + layouts_["EmotesTextData"] = {{{ "ID", 0 }, { "Text", 1 }}}; + + // Light.dbc + layouts_["Light"] = {{{ "ID", 0 }, { "MapID", 1 }, { "X", 2 }, { "Z", 3 }, { "Y", 4 }, + { "InnerRadius", 5 }, { "OuterRadius", 6 }, { "LightParamsID", 7 }, + { "LightParamsIDRain", 8 }, { "LightParamsIDUnderwater", 9 }}}; + + // LightParams.dbc + layouts_["LightParams"] = {{{ "LightParamsID", 0 }}}; + + // LightParamsBands.dbc (custom split from LightIntBand/LightFloatBand) + layouts_["LightParamsBands"] = {{{ "BlockIndex", 1 }, { "NumKeyframes", 2 }, + { "TimeKey0", 3 }, { "Value0", 19 }}}; + + // LightIntBand.dbc (same structure as LightParamsBands) + layouts_["LightIntBand"] = {{{ "BlockIndex", 1 }, { "NumKeyframes", 2 }, + { "TimeKey0", 3 }, { "Value0", 19 }}}; + + // LightFloatBand.dbc + layouts_["LightFloatBand"] = {{{ "BlockIndex", 1 }, { "NumKeyframes", 2 }, + { "TimeKey0", 3 }, { "Value0", 19 }}}; + + // WorldMapArea.dbc + layouts_["WorldMapArea"] = {{{ "ID", 0 }, { "MapID", 1 }, { "AreaID", 2 }, + { "AreaName", 3 }, { "LocLeft", 4 }, { "LocRight", 5 }, { "LocTop", 6 }, + { "LocBottom", 7 }, { "DisplayMapID", 8 }, { "ParentWorldMapID", 10 }}}; + + LOG_INFO("DBCLayout: loaded ", layouts_.size(), " WotLK default layouts"); +} + bool DBCLayout::loadFromJson(const std::string& path) { std::ifstream f(path); if (!f.is_open()) { diff --git a/src/pipeline/dbc_loader.cpp b/src/pipeline/dbc_loader.cpp index dd1d6f52..3b926419 100644 --- a/src/pipeline/dbc_loader.cpp +++ b/src/pipeline/dbc_loader.cpp @@ -32,11 +32,28 @@ bool DBCFile::load(const std::vector& dbcData) { return false; } - // Detect CSV format: starts with '#' + // Detect metadata CSV format: starts with '#' if (dbcData[0] == '#') { return loadCSV(dbcData); } + // Detect plain CSV format (column-name header row, e.g. "id,raceId,...\n21,1,0\n") + // Guard: binary WDBC files start with the four printable bytes "WDBC", so + // we must explicitly exclude them before checking for plain-text content. + if (std::isalpha(static_cast(dbcData[0])) && + (dbcData.size() < 4 || std::memcmp(dbcData.data(), "WDBC", 4) != 0)) { + bool isPlainText = true; + size_t checkLen = std::min(dbcData.size(), size_t(256)); + for (size_t i = 0; i < checkLen; ++i) { + uint8_t c = dbcData[i]; + if (c == '\n' || c == '\r') break; + if (c < 0x20 || c > 0x7E) { isPlainText = false; break; } + } + if (isPlainText) { + return loadPlainCSV(dbcData); + } + } + if (dbcData.size() < sizeof(DBCHeader)) { LOG_ERROR("DBC data too small for header"); return false; @@ -331,5 +348,93 @@ bool DBCFile::loadCSV(const std::vector& csvData) { return true; } +bool DBCFile::loadPlainCSV(const std::vector& csvData) { + std::string text(reinterpret_cast(csvData.data()), csvData.size()); + std::istringstream stream(text); + std::string line; + + // First line is the column-name header — use it only to count fields. + if (!std::getline(stream, line) || line.empty()) { + LOG_ERROR("Plain CSV DBC: missing header line"); + return false; + } + if (!line.empty() && line.back() == '\r') line.pop_back(); + + fieldCount = 1; + for (char c : line) { + if (c == ',') ++fieldCount; + } + if (fieldCount == 0) { + LOG_ERROR("Plain CSV DBC: invalid field count"); + return false; + } + + recordSize = fieldCount * 4; + + // Build a string block on the fly. Offset 0 = empty string (always present). + std::vector strBlock(1, 0); + + // Helper: intern a string into strBlock, return its offset. + auto internString = [&](const std::string& s) -> uint32_t { + if (s.empty()) return 0; + uint32_t offset = static_cast(strBlock.size()); + for (char c : s) strBlock.push_back(static_cast(c)); + strBlock.push_back(0); // null terminator + return offset; + }; + + // Parse data rows. Non-numeric cells are stored in strBlock. + std::vector> rows; + while (std::getline(stream, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.empty()) continue; + + std::vector row(fieldCount, 0); + uint32_t col = 0; + size_t pos = 0; + while (col < fieldCount && pos <= line.size()) { + size_t end = line.find(',', pos); + if (end == std::string::npos) end = line.size(); + std::string tok = trimAscii(line.substr(pos, end - pos)); + if (!tok.empty()) { + bool isNumeric = false; + try { + int64_t v = std::stoll(tok); + row[col] = static_cast(static_cast(v)); + isNumeric = true; + } catch (...) {} + if (!isNumeric) { + // Non-numeric → store as string in the string block. + row[col] = internString(tok); + } + } + pos = (end < line.size()) ? end + 1 : line.size() + 1; + ++col; + } + rows.push_back(std::move(row)); + } + + stringBlock = std::move(strBlock); + stringBlockSize = static_cast(stringBlock.size()); + + recordCount = static_cast(rows.size()); + recordData.resize(static_cast(recordCount) * recordSize); + for (uint32_t i = 0; i < recordCount; ++i) { + uint8_t* dst = recordData.data() + static_cast(i) * recordSize; + for (uint32_t f = 0; f < fieldCount; ++f) { + uint32_t val = rows[i][f]; + std::memcpy(dst + f * 4, &val, 4); + } + } + + loaded = true; + idCacheBuilt = false; + idToIndexCache.clear(); + + LOG_DEBUG("Loaded plain CSV DBC: ", recordCount, " records, ", fieldCount, + " fields, ", stringBlockSize, " string bytes"); + return true; +} + } // namespace pipeline } // namespace wowee diff --git a/tools/asset_extract/extractor.cpp b/tools/asset_extract/extractor.cpp index b4b6c9d5..f8eef9b6 100644 --- a/tools/asset_extract/extractor.cpp +++ b/tools/asset_extract/extractor.cpp @@ -524,7 +524,7 @@ bool Extractor::enumerateFiles(const Options& opts, for (auto it = archives.rbegin(); it != archives.rend(); ++it) { HANDLE hMpq = nullptr; - if (!SFileOpenArchive(it->path.c_str(), 0, 0, &hMpq)) { + if (!SFileOpenArchive(it->path.c_str(), 0, MPQ_OPEN_READ_ONLY, &hMpq)) { std::cerr << " Failed to open: " << it->path << "\n"; continue; } @@ -634,8 +634,10 @@ bool Extractor::run(const Options& opts) { std::vector threadHandles; for (const auto& ad : archives) { HANDLE h = nullptr; - if (SFileOpenArchive(ad.path.c_str(), 0, 0, &h)) { + if (SFileOpenArchive(ad.path.c_str(), 0, MPQ_OPEN_READ_ONLY, &h)) { threadHandles.push_back({h, ad.priority}); + } else { + std::cerr << "Worker: failed to open " << ad.path << " (err=" << GetLastError() << ")\n"; } } if (threadHandles.empty()) { diff --git a/tools/dbc_to_csv/main.cpp b/tools/dbc_to_csv/main.cpp index 514d5a40..74b12847 100644 --- a/tools/dbc_to_csv/main.cpp +++ b/tools/dbc_to_csv/main.cpp @@ -44,6 +44,10 @@ std::vector readFileBytes(const std::string& path) { // Check whether offset points to a plausible string in the string block. bool isValidStringOffset(const std::vector& stringBlock, uint32_t offset) { if (offset >= stringBlock.size()) return false; + // Must be at a string boundary: offset 0 or immediately after a null terminator. + // Without this, small integer field values (e.g. modelId=4) coincidentally + // land in the middle of an existing string and are falsely flagged as strings. + if (offset > 0 && stringBlock[offset - 1] != 0) return false; // Must be null-terminated within the block and contain only printable/whitespace bytes. for (size_t i = offset; i < stringBlock.size(); ++i) { uint8_t c = stringBlock[i]; @@ -75,7 +79,9 @@ std::set detectStringColumns(const DBCFile& dbc, // If no string block (or trivial size), no string columns. if (stringBlock.size() <= 1) return stringCols; - for (uint32_t col = 0; col < fieldCount; ++col) { + // Column 0 is always the numeric record ID in WDBC files — never a string, + // even if the uint32 value happens to be a valid string-block offset. + for (uint32_t col = 1; col < fieldCount; ++col) { bool allZeroOrValid = true; bool hasNonZero = false;