From 397034a750f5d958bd2f211545bc8a045fd2e416 Mon Sep 17 00:00:00 2001 From: Kelsi Date: Wed, 6 May 2026 11:00:20 -0700 Subject: [PATCH] feat(extract): incremental --upgrade-extract skips up-to-date sidecars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compares the source file's mtime against the sidecar's; if the sidecar is newer, the conversion is skipped and counted into stats.skipped. Re-running --upgrade-extract on a fully-converted tree is now nearly free (just an mtime check per file). asset_extract --upgrade-extract Data/expansions/wotlk Walking ... (first run) JSON (DBC→JSON) : 240 ok asset_extract --upgrade-extract Data/expansions/wotlk Walking ... (second run, all sidecars up to date) up-to-date (skip) : 240 JSON (DBC→JSON) : 0 ok emitOpenFormats() takes a new optional 'incremental' flag (default false to preserve the asset_extract main-loop's overwrite behavior since fresh extraction always wants new sidecars). Verified end-to-end with a hand-built DBC: first run converts, second run reports 'up-to-date (skip): 1'. --- tools/asset_extract/main.cpp | 7 +++- tools/asset_extract/open_format_emitter.cpp | 42 ++++++++++++++++++--- tools/asset_extract/open_format_emitter.hpp | 8 +++- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/tools/asset_extract/main.cpp b/tools/asset_extract/main.cpp index 3e36da97..f23edbe3 100644 --- a/tools/asset_extract/main.cpp +++ b/tools/asset_extract/main.cpp @@ -125,13 +125,18 @@ int main(int argc, char** argv) { wowee::tools::OpenFormatStats stats; // Pass 0 to auto-detect threads (or honor user --threads override). unsigned int t = opts.threads > 0 ? static_cast(opts.threads) : 0; + // upgrade-extract is always incremental — skip files whose sidecar + // is already up to date so re-runs are cheap. wowee::tools::emitOpenFormats(upgradeDir, opts.emitPng, opts.emitJsonDbc, opts.emitWom, opts.emitWob, - opts.emitTerrain, stats, t); + opts.emitTerrain, stats, t, + /*incremental=*/true); auto secs = std::chrono::duration_cast( std::chrono::steady_clock::now() - t0).count() / 1000.0; std::cout << " elapsed : " << secs << " s\n"; + if (stats.skipped > 0) + std::cout << " up-to-date (skip) : " << stats.skipped << "\n"; std::cout << " PNG (BLP→PNG) : " << stats.pngOk << " ok" << (stats.pngFail ? ", " + std::to_string(stats.pngFail) + " failed" : "") << "\n"; std::cout << " JSON (DBC→JSON) : " << stats.jsonDbcOk << " ok" diff --git a/tools/asset_extract/open_format_emitter.cpp b/tools/asset_extract/open_format_emitter.cpp index c470bc29..6817e7f3 100644 --- a/tools/asset_extract/open_format_emitter.cpp +++ b/tools/asset_extract/open_format_emitter.cpp @@ -290,7 +290,20 @@ void emitOpenFormats(const std::string& rootDir, bool emitWom, bool emitWob, bool emitTerrain, OpenFormatStats& stats, - unsigned int threadCount) { + unsigned int threadCount, + bool incremental) { + // Returns true if `sidecarPath` exists and its mtime is >= source mtime. + // Used by the incremental walk to skip up-to-date conversions. + auto sidecarUpToDate = [](const std::string& sourcePath, + const std::string& sidecarPath) { + std::error_code ec; + if (!fs::exists(sidecarPath, ec)) return false; + auto srcTime = fs::last_write_time(sourcePath, ec); + if (ec) return false; + auto sideTime = fs::last_write_time(sidecarPath, ec); + if (ec) return false; + return sideTime >= srcTime; + }; if (!fs::exists(rootDir)) return; if (!emitPng && !emitJsonDbc && !emitWom && !emitWob && !emitTerrain) return; @@ -314,18 +327,35 @@ void emitOpenFormats(const std::string& rootDir, base = base.substr(0, base.size() - ext.size()); std::string p = entry.path().string(); - if (emitPng && ext == ".blp") jobs.push_back({p, base, Kind::Png}); - else if (emitJsonDbc && ext == ".dbc") jobs.push_back({p, base, Kind::JsonDbc}); - else if (emitWom && ext == ".m2") jobs.push_back({p, base, Kind::Wom}); + // For incremental, skip the job entirely if the sidecar already + // tracks the source. For terrain we treat .whm as the canonical + // sidecar (the WHM/WOT/WOC trio always written together). + auto skipIfFresh = [&](const std::string& sidecar) -> bool { + if (!incremental) return false; + if (sidecarUpToDate(p, sidecar)) { stats.skipped++; return true; } + return false; + }; + if (emitPng && ext == ".blp") { + if (!skipIfFresh(base + ".png")) jobs.push_back({p, base, Kind::Png}); + } + else if (emitJsonDbc && ext == ".dbc") { + if (!skipIfFresh(base + ".json")) jobs.push_back({p, base, Kind::JsonDbc}); + } + else if (emitWom && ext == ".m2") { + if (!skipIfFresh(base + ".wom")) jobs.push_back({p, base, Kind::Wom}); + } else if (emitWob && ext == ".wmo") { // Skip group sub-files (_NNN.wmo) — merged into root WMO. std::string fname = entry.path().filename().string(); auto under = fname.rfind('_'); bool isGroup = (under != std::string::npos && fname.size() - under == 8); - if (!isGroup) jobs.push_back({p, base, Kind::Wob}); + if (!isGroup && !skipIfFresh(base + ".wob")) + jobs.push_back({p, base, Kind::Wob}); + } + else if (emitTerrain && ext == ".adt") { + if (!skipIfFresh(base + ".whm")) jobs.push_back({p, base, Kind::Terrain}); } - else if (emitTerrain && ext == ".adt") jobs.push_back({p, base, Kind::Terrain}); } if (jobs.empty()) return; diff --git a/tools/asset_extract/open_format_emitter.hpp b/tools/asset_extract/open_format_emitter.hpp index 0a2b0b18..60d50c43 100644 --- a/tools/asset_extract/open_format_emitter.hpp +++ b/tools/asset_extract/open_format_emitter.hpp @@ -22,6 +22,9 @@ struct OpenFormatStats { uint32_t wobOk = 0, wobFail = 0; uint32_t whmOk = 0, whmFail = 0; uint32_t wocOk = 0, wocFail = 0; + // Files where the sidecar already existed and was newer than the + // proprietary source — skipped (incremental mode). + uint32_t skipped = 0; }; // Convert one BLP file on disk to a PNG side-file. @@ -51,12 +54,15 @@ bool emitTerrainFromAdt(const std::string& adtPath, const std::string& outBase); // Walk an extracted-asset directory and emit open-format side-files for // every requested format. Counts accumulated into stats. // `threadCount` 0 = auto-detect from hardware_concurrency(). +// If `incremental` is true, files whose sidecar already exists and is +// newer than the proprietary source are skipped (counted in stats.skipped). void emitOpenFormats(const std::string& rootDir, bool emitPng, bool emitJsonDbc, bool emitWom, bool emitWob, bool emitTerrain, OpenFormatStats& stats, - unsigned int threadCount = 0); + unsigned int threadCount = 0, + bool incremental = false); } // namespace tools } // namespace wowee