From 24199e916d5b270a0d2bcfd953eaeaef042918c0 Mon Sep 17 00:00:00 2001 From: Mike Klaas Date: Sat, 4 Jul 2026 14:58:44 -0700 Subject: [PATCH] Grow ability for `ce-dat-tool` to extract a list of files (#506) * Grow ability for `ce-dat-tool` to extract a list of files This is needed for Et Tu installation, which selectively extracts files from fo1's master.dat. (It doesn't work if everything is extracted) * cache entry list to avoid O(n) cost when extracting many files --- CONTRIBUTING.md | 5 +- tools/dat_archive.cc | 38 +++++++++++++--- tools/dat_archive.h | 7 +++ tools/dat_tool.cc | 106 ++++++++++++++++++++++++++++++++++++++----- 4 files changed, 136 insertions(+), 20 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fd3bd7e8..a2151212 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,8 +66,9 @@ Available commands: 1. `.//ce-dat-tool list [pattern]` 2. `ce-dat-tool info [pattern]` 3. `ce-dat-tool extract [--lower] [pattern]` -4. `ce-dat-tool cat ` -5. `ce-dat-tool create ` +4. `ce-dat-tool extract [--lower] (--file-list|--files-from) ` +5. `ce-dat-tool cat ` +6. `ce-dat-tool create ` Use `--lower` with `extract` when you want every extracted file and directory name forced to lowercase. For example: diff --git a/tools/dat_archive.cc b/tools/dat_archive.cc index cc038e4b..941c9e31 100644 --- a/tools/dat_archive.cc +++ b/tools/dat_archive.cc @@ -47,9 +47,17 @@ std::string normalizeDatPath(std::string path) return path; } -bool normalizeAndComparePath(const std::string& a, const std::string& b) +std::string normalizeDatPathKey(std::string path) { - return compat_stricmp(normalizeDatPath(a).c_str(), normalizeDatPath(b).c_str()) == 0; + for (char& ch : path) { + if (ch == '/') { + ch = '\\'; + } else { + ch = static_cast(std::tolower(static_cast(ch))); + } + } + + return path; } bool readBe32(FILE* stream, int* value) @@ -539,16 +547,32 @@ private: const DatArchiveEntry* DatArchive::findEntry(const std::string& path) const { - std::string normalizedPath = normalizeDatPath(path); - for (const DatArchiveEntry& entry : entries()) { - if (normalizeAndComparePath(entry.path, normalizedPath)) { - return &entry; - } + ensureEntryLookup(); + + auto it = entryLookup_.find(normalizeDatPathKey(path)); + if (it != entryLookup_.end()) { + return it->second; } return nullptr; } +void DatArchive::ensureEntryLookup() const +{ + if (entryLookupInitialized_) { + return; + } + + const std::vector& archiveEntries = entries(); + entryLookup_.reserve(archiveEntries.size()); + + for (const DatArchiveEntry& entry : entries()) { + entryLookup_.emplace(normalizeDatPathKey(entry.path), &entry); + } + + entryLookupInitialized_ = true; +} + std::vector DatArchive::findEntries(const std::string& pattern) const { std::vector matches; diff --git a/tools/dat_archive.h b/tools/dat_archive.h index 6aff00e5..2c7b75f3 100644 --- a/tools/dat_archive.h +++ b/tools/dat_archive.h @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace fallout { @@ -42,6 +43,12 @@ public: std::vector findEntries(const std::string& pattern) const; static std::unique_ptr open(const std::string& path); + +private: + void ensureEntryLookup() const; + + mutable bool entryLookupInitialized_ = false; + mutable std::unordered_map entryLookup_; }; } // namespace fallout diff --git a/tools/dat_tool.cc b/tools/dat_tool.cc index d6c7bf62..5923ca11 100644 --- a/tools/dat_tool.cc +++ b/tools/dat_tool.cc @@ -44,6 +44,7 @@ struct Options { std::string archivePath; std::string command; std::vector args; + std::string extractFileListPath; bool lowerExtractedPaths = false; }; @@ -105,6 +106,21 @@ std::string toLowerAscii(std::string value) return value; } +std::string trimAsciiWhitespace(std::string_view value) +{ + const auto isNotSpace = [](char ch) { + return !std::isspace(static_cast(ch)); + }; + + const auto start = std::find_if(value.begin(), value.end(), isNotSpace); + if (start == value.end()) { + return {}; + } + + const auto end = std::find_if(value.rbegin(), value.rend(), isNotSpace).base(); + return std::string(start, end); +} + bool isAbsoluteOutputPath(const std::string& path) { if (path.empty()) { @@ -789,12 +805,14 @@ void printUsage(std::ostream& stream) << " ce-dat-tool list [pattern]\n" << " ce-dat-tool info [pattern]\n" << " ce-dat-tool extract [--lower] [pattern]\n" + << " ce-dat-tool extract [--lower] (--file-list|--files-from) \n" << " ce-dat-tool cat \n" << "\n" << "Notes:\n" << " - Create preserves input path casing and stores Windows-style archive paths.\n" << " - Create compresses entries only when zlib output is smaller.\n" << " - Patterns use the same Windows-style wildcard matching as the game.\n" + << " - File lists contain one archive path per line. Empty lines are ignored.\n" << " - Archive paths are case-insensitive and should use backslashes internally.\n"; } @@ -819,11 +837,23 @@ bool parseOptions(int argc, char** argv, Options* options) } if (options->command == "extract") { - auto lowerIt = std::find(options->args.begin(), options->args.end(), "--lower"); - if (lowerIt != options->args.end()) { - options->lowerExtractedPaths = true; - options->args.erase(lowerIt); + std::vector args; + for (size_t index = 0; index < options->args.size(); index++) { + const std::string& arg = options->args[index]; + if (arg == "--lower") { + options->lowerExtractedPaths = true; + } else if (arg == "--file-list" || arg == "--files-from") { + if (index + 1 >= options->args.size()) { + return false; + } + + options->extractFileListPath = options->args[index + 1]; + index++; + } else { + args.push_back(arg); + } } + options->args = std::move(args); } return true; @@ -897,6 +927,37 @@ int listCommand(const DatArchive& archive, const std::string& pattern) return 0; } +bool readExtractFileList(const std::string& fileListPath, std::vector* paths) +{ + std::ifstream input(fileListPath); + if (!input.is_open()) { + std::cerr << "Failed to open file list: " << fileListPath << "\n"; + return false; + } + + std::string line; + while (std::getline(input, line)) { + std::string path = trimAsciiWhitespace(line); + if (path.empty()) { + continue; + } + + paths->push_back(normalizeDatPath(std::move(path))); + } + + if (!input.eof()) { + std::cerr << "Failed while reading file list: " << fileListPath << "\n"; + return false; + } + + if (paths->empty()) { + std::cerr << "File list is empty: " << fileListPath << "\n"; + return false; + } + + return true; +} + int infoCommand(const DatArchive& archive, const std::vector& args) { if (args.empty()) { @@ -954,7 +1015,7 @@ int infoCommand(const DatArchive& archive, const std::vector& args) return 0; } -int extractCommand(const DatArchive& archive, const std::vector& args, bool lowerExtractedPaths) +int extractCommand(const DatArchive& archive, const std::vector& args, bool lowerExtractedPaths, const std::string& fileListPath) { if (args.empty()) { std::cerr << "extract requires an output directory\n"; @@ -963,7 +1024,12 @@ int extractCommand(const DatArchive& archive, const std::vector& ar std::string outputDir = args[0]; std::string pattern = "*"; - if (args.size() >= 2) { + if (!fileListPath.empty() && args.size() >= 2) { + std::cerr << "extract cannot use both --file-list and a pattern\n"; + return 1; + } + + if (fileListPath.empty() && args.size() >= 2) { pattern = normalizeDatPath(args[1]); } @@ -972,10 +1038,28 @@ int extractCommand(const DatArchive& archive, const std::vector& ar return 1; } - const std::vector matches = archive.findEntries(pattern); - if (matches.empty()) { - std::cerr << "No entries matched pattern: " << pattern << "\n"; - return 1; + std::vector matches; + if (!fileListPath.empty()) { + std::vector fileListPaths; + if (!readExtractFileList(fileListPath, &fileListPaths)) { + return 1; + } + + for (const std::string& entryPath : fileListPaths) { + const DatArchiveEntry* entry = archive.findEntry(entryPath); + if (entry == nullptr) { + std::cerr << "Entry not found: " << entryPath << "\n"; + return 1; + } + + matches.push_back(entry); + } + } else { + matches = archive.findEntries(pattern); + if (matches.empty()) { + std::cerr << "No entries matched pattern: " << pattern << "\n"; + return 1; + } } int extracted = 0; @@ -1065,7 +1149,7 @@ int run(const Options& options) } else if (options.command == "info") { rc = infoCommand(*archive, options.args); } else if (options.command == "extract") { - rc = extractCommand(*archive, options.args, options.lowerExtractedPaths); + rc = extractCommand(*archive, options.args, options.lowerExtractedPaths, options.extractFileListPath); } else if (options.command == "cat") { rc = catCommand(*archive, options.args); } else {