feat(extract): incremental --upgrade-extract skips up-to-date sidecars

Compares the source file's mtime against the sidecar's; if the
sidecar is newer, the conversion is skipped and counted into
stats.skipped. Re-running --upgrade-extract on a fully-converted
tree is now nearly free (just an mtime check per file).

  asset_extract --upgrade-extract Data/expansions/wotlk
  Walking ... (first run)
    JSON (DBC→JSON)   : 240 ok
  asset_extract --upgrade-extract Data/expansions/wotlk
  Walking ... (second run, all sidecars up to date)
    up-to-date (skip) : 240
    JSON (DBC→JSON)   : 0 ok

emitOpenFormats() takes a new optional 'incremental' flag (default
false to preserve the asset_extract main-loop's overwrite behavior
since fresh extraction always wants new sidecars).

Verified end-to-end with a hand-built DBC: first run converts,
second run reports 'up-to-date (skip): 1'.
This commit is contained in:
Kelsi 2026-05-06 11:00:20 -07:00
parent 463a8cd751
commit 397034a750
3 changed files with 49 additions and 8 deletions

View file

@ -125,13 +125,18 @@ int main(int argc, char** argv) {
wowee::tools::OpenFormatStats stats; wowee::tools::OpenFormatStats stats;
// Pass 0 to auto-detect threads (or honor user --threads override). // Pass 0 to auto-detect threads (or honor user --threads override).
unsigned int t = opts.threads > 0 ? static_cast<unsigned int>(opts.threads) : 0; unsigned int t = opts.threads > 0 ? static_cast<unsigned int>(opts.threads) : 0;
// upgrade-extract is always incremental — skip files whose sidecar
// is already up to date so re-runs are cheap.
wowee::tools::emitOpenFormats(upgradeDir, wowee::tools::emitOpenFormats(upgradeDir,
opts.emitPng, opts.emitJsonDbc, opts.emitPng, opts.emitJsonDbc,
opts.emitWom, opts.emitWob, opts.emitWom, opts.emitWob,
opts.emitTerrain, stats, t); opts.emitTerrain, stats, t,
/*incremental=*/true);
auto secs = std::chrono::duration_cast<std::chrono::milliseconds>( auto secs = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - t0).count() / 1000.0; std::chrono::steady_clock::now() - t0).count() / 1000.0;
std::cout << " elapsed : " << secs << " s\n"; std::cout << " elapsed : " << secs << " s\n";
if (stats.skipped > 0)
std::cout << " up-to-date (skip) : " << stats.skipped << "\n";
std::cout << " PNG (BLP→PNG) : " << stats.pngOk << " ok" std::cout << " PNG (BLP→PNG) : " << stats.pngOk << " ok"
<< (stats.pngFail ? ", " + std::to_string(stats.pngFail) + " failed" : "") << "\n"; << (stats.pngFail ? ", " + std::to_string(stats.pngFail) + " failed" : "") << "\n";
std::cout << " JSON (DBC→JSON) : " << stats.jsonDbcOk << " ok" std::cout << " JSON (DBC→JSON) : " << stats.jsonDbcOk << " ok"

View file

@ -290,7 +290,20 @@ void emitOpenFormats(const std::string& rootDir,
bool emitWom, bool emitWob, bool emitWom, bool emitWob,
bool emitTerrain, bool emitTerrain,
OpenFormatStats& stats, OpenFormatStats& stats,
unsigned int threadCount) { unsigned int threadCount,
bool incremental) {
// Returns true if `sidecarPath` exists and its mtime is >= source mtime.
// Used by the incremental walk to skip up-to-date conversions.
auto sidecarUpToDate = [](const std::string& sourcePath,
const std::string& sidecarPath) {
std::error_code ec;
if (!fs::exists(sidecarPath, ec)) return false;
auto srcTime = fs::last_write_time(sourcePath, ec);
if (ec) return false;
auto sideTime = fs::last_write_time(sidecarPath, ec);
if (ec) return false;
return sideTime >= srcTime;
};
if (!fs::exists(rootDir)) return; if (!fs::exists(rootDir)) return;
if (!emitPng && !emitJsonDbc && !emitWom && !emitWob && !emitTerrain) return; if (!emitPng && !emitJsonDbc && !emitWom && !emitWob && !emitTerrain) return;
@ -314,18 +327,35 @@ void emitOpenFormats(const std::string& rootDir,
base = base.substr(0, base.size() - ext.size()); base = base.substr(0, base.size() - ext.size());
std::string p = entry.path().string(); std::string p = entry.path().string();
if (emitPng && ext == ".blp") jobs.push_back({p, base, Kind::Png}); // For incremental, skip the job entirely if the sidecar already
else if (emitJsonDbc && ext == ".dbc") jobs.push_back({p, base, Kind::JsonDbc}); // tracks the source. For terrain we treat .whm as the canonical
else if (emitWom && ext == ".m2") jobs.push_back({p, base, Kind::Wom}); // sidecar (the WHM/WOT/WOC trio always written together).
auto skipIfFresh = [&](const std::string& sidecar) -> bool {
if (!incremental) return false;
if (sidecarUpToDate(p, sidecar)) { stats.skipped++; return true; }
return false;
};
if (emitPng && ext == ".blp") {
if (!skipIfFresh(base + ".png")) jobs.push_back({p, base, Kind::Png});
}
else if (emitJsonDbc && ext == ".dbc") {
if (!skipIfFresh(base + ".json")) jobs.push_back({p, base, Kind::JsonDbc});
}
else if (emitWom && ext == ".m2") {
if (!skipIfFresh(base + ".wom")) jobs.push_back({p, base, Kind::Wom});
}
else if (emitWob && ext == ".wmo") { else if (emitWob && ext == ".wmo") {
// Skip group sub-files (<base>_NNN.wmo) — merged into root WMO. // Skip group sub-files (<base>_NNN.wmo) — merged into root WMO.
std::string fname = entry.path().filename().string(); std::string fname = entry.path().filename().string();
auto under = fname.rfind('_'); auto under = fname.rfind('_');
bool isGroup = (under != std::string::npos && bool isGroup = (under != std::string::npos &&
fname.size() - under == 8); fname.size() - under == 8);
if (!isGroup) jobs.push_back({p, base, Kind::Wob}); if (!isGroup && !skipIfFresh(base + ".wob"))
jobs.push_back({p, base, Kind::Wob});
}
else if (emitTerrain && ext == ".adt") {
if (!skipIfFresh(base + ".whm")) jobs.push_back({p, base, Kind::Terrain});
} }
else if (emitTerrain && ext == ".adt") jobs.push_back({p, base, Kind::Terrain});
} }
if (jobs.empty()) return; if (jobs.empty()) return;

View file

@ -22,6 +22,9 @@ struct OpenFormatStats {
uint32_t wobOk = 0, wobFail = 0; uint32_t wobOk = 0, wobFail = 0;
uint32_t whmOk = 0, whmFail = 0; uint32_t whmOk = 0, whmFail = 0;
uint32_t wocOk = 0, wocFail = 0; uint32_t wocOk = 0, wocFail = 0;
// Files where the sidecar already existed and was newer than the
// proprietary source — skipped (incremental mode).
uint32_t skipped = 0;
}; };
// Convert one BLP file on disk to a PNG side-file. // Convert one BLP file on disk to a PNG side-file.
@ -51,12 +54,15 @@ bool emitTerrainFromAdt(const std::string& adtPath, const std::string& outBase);
// Walk an extracted-asset directory and emit open-format side-files for // Walk an extracted-asset directory and emit open-format side-files for
// every requested format. Counts accumulated into stats. // every requested format. Counts accumulated into stats.
// `threadCount` 0 = auto-detect from hardware_concurrency(). // `threadCount` 0 = auto-detect from hardware_concurrency().
// If `incremental` is true, files whose sidecar already exists and is
// newer than the proprietary source are skipped (counted in stats.skipped).
void emitOpenFormats(const std::string& rootDir, void emitOpenFormats(const std::string& rootDir,
bool emitPng, bool emitJsonDbc, bool emitPng, bool emitJsonDbc,
bool emitWom, bool emitWob, bool emitWom, bool emitWob,
bool emitTerrain, bool emitTerrain,
OpenFormatStats& stats, OpenFormatStats& stats,
unsigned int threadCount = 0); unsigned int threadCount = 0,
bool incremental = false);
} // namespace tools } // namespace tools
} // namespace wowee } // namespace wowee