From 38cecac07df1f73628be0db5bb5a97b017d21a2b Mon Sep 17 00:00:00 2001 From: Oliver Hamlet Date: Fri, 16 Dec 2022 11:43:09 +0000 Subject: [PATCH] Add support for checking assets in BA2 files They're used by Fallout 4. BA2 file and folder hashes are 32-bit, so collisions are much more likely than with BSAs, which use 64-bit hashes. I put in some logging to check the likelihood of collisions, and found that the largest number of assets loaded was by Fallout4.esm (no surprise), which loaded 371182 files across 5746 folders, with the most files in one folder being 124871. That puts the probability of a hash collision within that folder at above 80%. I did see different Fallout4 -*.ba2 files contain files with the same combination of folder and file hashes, and similar for DLCUltraHighResolution -*.ba2 files, so I'm going to try calculating 64-bit hashes from the file paths stored in the BA2 files. --- CMakeLists.txt | 4 +- docs/api/sorting.rst | 6 +- src/api/bsa.cpp | 308 +++++++++++++++++++++----- src/api/bsa_detail.h | 22 +- src/tests/api/internals/bsa_test.h | 40 ++++ src/tests/api/internals/plugin_test.h | 22 +- 6 files changed, 328 insertions(+), 74 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dde651c5..93ab9b9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,8 +122,8 @@ endif() ExternalProject_Add(testing-plugins PREFIX "external" - URL "https://github.com/Ortham/testing-plugins/archive/1.4.1.tar.gz" - URL_HASH "SHA256=57f82495a737013c1b828d43fbe87edcb57768f85c6416e8d17f273d692691cf" + URL "https://github.com/Ortham/testing-plugins/archive/1.5.0.tar.gz" + URL_HASH "SHA256=98c9094fb0f0152b1af7a6206950207f7ddc6602cd44ed67ebf70603ef490791" CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "") diff --git a/docs/api/sorting.rst b/docs/api/sorting.rst index 02929cd2..6161c579 100644 --- a/docs/api/sorting.rst +++ b/docs/api/sorting.rst @@ -59,9 +59,9 @@ can be corrected. Plugin overlap edges are then added. Two plugins overlap if they contain the same record, i.e. if they both edit the same record or if one edits a record the -other plugin adds. Plugins also overlap if they both load one or more BSAs and -the BSAs loaded by one plugin contain data for a file path that is also included -in the BSAs loaded by the other plugin. +other plugin adds. Plugins also overlap if they both load one or more BSAs (BA2s +for Fallout 4) and the BSAs loaded by one plugin contain data for a file path +that is also included in the BSAs loaded by the other plugin. For each plugin, skip it if it overrides no records, otherwise iterate over all other plugins. diff --git a/src/api/bsa.cpp b/src/api/bsa.cpp index 1fe3005b..e18584af 100644 --- a/src/api/bsa.cpp +++ b/src/api/bsa.cpp @@ -39,28 +39,31 @@ BSA format documentation: */ constexpr std::array BSA_TYPE_ID = {'B', 'S', 'A', '\0'}; +constexpr std::array BA2_TYPE_ID = {'B', 'T', 'D', 'X'}; +constexpr std::array BA2_GENERAL_TYPE = {'G', 'N', 'R', 'L'}; +constexpr std::array BA2_TEXTURE_TYPE = {'D', 'X', '1', '0'}; -namespace bsa::v103 { +namespace bsa { +namespace v103 { struct FolderRecord { uint64_t nameHash{0}; uint32_t fileCount{0}; uint32_t fileRecordsOffset{0}; }; -std::map> GetAssetsInBethesdaArchive( - std::istream& in, - const Header& header) { - return detail::GetAssetsInBethesdaArchive(in, header); +std::map> GetAssetsInBSA(std::istream& in, + const Header& header) { + return detail::GetAssetsInBSA(in, header); } } -namespace bsa::v104 { -using bsa::v103::FolderRecord; +namespace v104 { +using v103::FolderRecord; -using v103::GetAssetsInBethesdaArchive; +using v103::GetAssetsInBSA; } -namespace bsa::v105 { +namespace v105 { struct FolderRecord { uint64_t nameHash{0}; uint32_t fileCount{0}; @@ -69,10 +72,232 @@ struct FolderRecord { uint32_t padding2{0}; }; -std::map> GetAssetsInBethesdaArchive( +std::map> GetAssetsInBSA(std::istream& in, + const Header& header) { + return detail::GetAssetsInBSA(in, header); +} +} + +std::map> GetAssetsInBSA( std::istream& in, - const Header& header) { - return detail::GetAssetsInBethesdaArchive(in, header); + const bsa::Header& header) { + const auto logger = getLogger(); + + // Validate the header. + if (header.typeId != BSA_TYPE_ID || + !(header.version == 103 || header.version == 104 || + header.version == 105) || + header.recordsOffset != 36) { + throw std::runtime_error("BSA file has an invalid header"); + } + + if ((header.archiveFlags & 0x40) != 0) { + throw std::runtime_error("BSA file uses big-endian numbers"); + } + + if (header.version == 103) { + return bsa::v103::GetAssetsInBSA(in, header); + } + + if (header.version == 104) { + return bsa::v104::GetAssetsInBSA(in, header); + } + + if (header.version == 105) { + return bsa::v105::GetAssetsInBSA(in, header); + } + + throw std::runtime_error("BSA file has an unrecognised version"); +} +} + +namespace ba2 { +struct Header { + std::array typeId; + uint32_t version{0}; + std::array archiveType; + uint32_t fileCount{0}; + uint64_t filePathsOffset{0}; +}; + +struct GeneralFileRecord { + uint32_t fileHash{ + 0}; // Seems to be a hash of the file basename (without extension) + uint32_t extension{0}; // First 4 bytes of file extension. + uint32_t folderHash{0}; // Same value for files in the same directory. + uint32_t unknown1{0}; // Always 00 01 10 00? + uint32_t dataOffset{0}; // Offset from start of file. + uint32_t unknown2{0}; // Always null? + uint32_t unknown3{ + 0}; // Length of the compressed data block (or null if uncompressed). + uint32_t dataLength{0}; // Length of the uncompressed data block. + uint32_t unknown4{0}; // Always 0D F0 AD BA? +}; + +struct TextureFileRecord { + uint32_t fileHash{0}; + uint32_t extension{0}; + uint32_t folderHash{0}; + uint8_t unknown1{0}; // Always null? + uint8_t subrecordCount{0}; + uint16_t subrecordLength{0}; // Length of a single subrecord, always 24? + uint64_t unknown2{0}; +}; + +void StoreHashes(std::map>& folderFileHashes, + const uint32_t fileHash, + const uint32_t extension, + const uint32_t folderHash) { + // The file hash can't be used on its own as it's common for two files + // in the same directory to have the same basename but different + // extension, so treat the extension as the high bytes of a 64-bit hash. + const uint64_t fileAndExtensionHash = + uint64_t{fileHash} | (uint64_t{extension} << 32); + + const auto folderResult = folderFileHashes.emplace( + folderHash, std::set({fileAndExtensionHash})); + + if (!folderResult.second) { + // Folder hash already stored, add file hash to existing set. + const auto fileResult = + folderResult.first->second.insert(fileAndExtensionHash); + + if (!fileResult.second) { + const auto message = fmt::format( + "Unexpected collision for file name hash {:x} in set for folder name " + "hash {:x}", + fileAndExtensionHash, + folderHash); + throw std::runtime_error(message); + } + } +} + +std::map> GetAssetsInGeneralBA2( + std::istream& in, + const Header& header, + const std::filesystem::path& archivePath) { + const auto logger = getLogger(); + + std::map> folderFileHashes; + + for (size_t i = 0; i < header.fileCount; ++i) { + GeneralFileRecord fileRecord; + in.read(reinterpret_cast(&fileRecord), sizeof(GeneralFileRecord)); + + // Validate assumptions (don't error if invalid, because they don't + // actually matter for LOOT). + if (fileRecord.unknown1 != 0x100100) { + logger->warn( + "Unexpected value for unknown1 field in file record with hash {:x} " + "in BA2 file at \"{}\": {}", + fileRecord.fileHash, + archivePath.u8string(), + fileRecord.unknown1); + } + + if (fileRecord.unknown2 != 0) { + logger->warn( + "Unexpected value for unknown2 field in file record with hash {:x} " + "in BA2 file at \"{}\": {}", + fileRecord.fileHash, + archivePath.u8string(), + fileRecord.unknown2); + } + + if (fileRecord.unknown4 != 0xBAADF00D) { + logger->warn( + "Unexpected value for unknown4 field in file record with hash {:x} " + "in BA2 file at \"{}\": {}", + fileRecord.fileHash, + archivePath.u8string(), + fileRecord.unknown4); + } + + // Now store the hashes. + StoreHashes(folderFileHashes, + fileRecord.fileHash, + fileRecord.extension, + fileRecord.folderHash); + } + + return folderFileHashes; +} + +std::map> GetAssetsInTextureBA2( + std::istream& in, + const Header& header, + const std::filesystem::path& archivePath) { + const auto logger = getLogger(); + + std::map> folderFileHashes; + + for (size_t i = 0; i < header.fileCount; ++i) { + TextureFileRecord fileRecord; + in.read(reinterpret_cast(&fileRecord), sizeof(TextureFileRecord)); + + // Skip over this file record's subrecords. + in.ignore(fileRecord.subrecordCount * fileRecord.subrecordLength); + + // Validate assumptions (don't error if invalid, because they don't + // actually matter for LOOT). + if (fileRecord.unknown1 != 0) { + logger->warn( + "Unexpected value for unknown1 field in file record with hash {:x} " + "in BA2 file at \"{}\": {}", + fileRecord.fileHash, + archivePath.u8string(), + fileRecord.unknown1); + } + + if (fileRecord.subrecordLength != 24) { + logger->warn( + "Unexpected value for subrecordLength field in file record with hash " + "{:x} in BA2 file at \"{}\": {}", + fileRecord.fileHash, + archivePath.u8string(), + fileRecord.subrecordLength); + } + + // Now store the hashes. + StoreHashes(folderFileHashes, + fileRecord.fileHash, + fileRecord.extension, + fileRecord.folderHash); + } + + return folderFileHashes; +} + +std::map> GetAssetsInBA2( + std::istream& in, + const Header& header, + const std::filesystem::path& archivePath) { + // Validate the header. + if (header.typeId != BA2_TYPE_ID) { + throw std::runtime_error("BA2 file header type ID is invalid"); + } + + if (header.version != 1) { + throw std::runtime_error("BA2 file header version is invalid"); + } + + if (header.archiveType != BA2_GENERAL_TYPE && + header.archiveType != BA2_TEXTURE_TYPE) { + throw std::runtime_error("BA2 file header archive type is invalid"); + } + + // BA2s don't have the same structure as BSAs, for general BA2s everything is + // stored in a flatter structure, and for texture BA2s a more specialised + // structure is used. While there are still directory and file hashes, they're + // 32-bit, not 64-bit, which makes collisions more likely. Still, try to use + // them for simplicity. + + if (header.archiveType == BA2_GENERAL_TYPE) { + return GetAssetsInGeneralBA2(in, header, archivePath); + } + + return GetAssetsInTextureBA2(in, header, archivePath); } } @@ -104,8 +329,6 @@ std::map> GetAssetsInBethesdaArchive( if (!std::filesystem::exists(archivePath)) { if (logger) { - logger->error("Bethesda archive path \"{}\" does not exist!", - archivePath.u8string()); throw std::runtime_error("Bethesda archive does not exist"); } } @@ -115,47 +338,32 @@ std::map> GetAssetsInBethesdaArchive( std::ios::eofbit); // Causes ifstream::failure to be thrown if // a problem is encountered. - bsa::Header header; - in.read(reinterpret_cast(&header), sizeof(bsa::Header)); + std::array typeId; + in.read(typeId.data(), typeId.size()); - // Validate the header. - if (header.typeId != BSA_TYPE_ID || - !(header.version == 103 || header.version == 104 || - header.version == 105) || - header.recordsOffset != 36) { - if (logger) { - logger->error("Bethesda archive at \"{}\" has an invalid header.", - archivePath.u8string()); - throw std::runtime_error("Bethesda archive has an invalid header"); - } + if (typeId == BSA_TYPE_ID) { + bsa::Header header; + header.typeId = typeId; + + // Read the rest of the header. + in.read(reinterpret_cast(&header) + typeId.size(), + sizeof(bsa::Header) - typeId.size()); + + return bsa::GetAssetsInBSA(in, header); } - if ((header.archiveFlags & 0x40) != 0) { - if (logger) { - logger->error("BSA file at \"{}\" uses big-endian numbers."); - } - throw std::runtime_error("BSA file uses big-endian numbers"); + if (typeId == BA2_TYPE_ID) { + ba2::Header header; + header.typeId = typeId; + + // Read the rest of the header. + in.read(reinterpret_cast(&header) + typeId.size(), + sizeof(ba2::Header) - typeId.size()); + + return ba2::GetAssetsInBA2(in, header, archivePath); } - if (header.version == 103) { - return bsa::v103::GetAssetsInBethesdaArchive(in, header); - } - - if (header.version == 104) { - return bsa::v104::GetAssetsInBethesdaArchive(in, header); - } - - if (header.version == 105) { - return bsa::v105::GetAssetsInBethesdaArchive(in, header); - } - - if (logger) { - logger->error("Unrecognised BSA version {} in archive at \"{}\"", - header.version, - archivePath.u8string()); - } - - throw std::runtime_error("BSA file has an unrecognised version"); + throw std::runtime_error("Bethesda archive has unrecognised typeId"); } std::map> GetAssetsInBethesdaArchives( diff --git a/src/api/bsa_detail.h b/src/api/bsa_detail.h index a851a17f..271f7d06 100644 --- a/src/api/bsa_detail.h +++ b/src/api/bsa_detail.h @@ -56,9 +56,8 @@ struct FileRecord { namespace loot::bsa::detail { template -std::map> GetAssetsInBethesdaArchive( - std::istream& in, - const Header& header) { +std::map> GetAssetsInBSA(std::istream& in, + const Header& header) { const auto logger = getLogger(); std::vector folderRecords(header.folderCount); @@ -93,10 +92,8 @@ std::map> GetAssetsInBethesdaArchive( folderFileHashes.emplace(folderHash, std::set()); if (!folderResult.second) { - if (logger) { - logger->warn("Folder name hash {} is already in map", folderHash); - } - throw std::runtime_error("Unexpected folder name hash collision"); + throw std::runtime_error("Unexpected collision for folder name hash " + + std::to_string(folderHash)); } size_t fileRecordsOffset = 0; @@ -126,13 +123,10 @@ std::map> GetAssetsInBethesdaArchive( folderResult.first->second.insert(fileRecord->nameHash); if (!result.second) { - if (logger) { - logger->warn( - "File name hash {} is already in the set for folder name hash {}", - fileRecord->nameHash, - folderHash); - } - throw std::runtime_error("Unexpected file name hash collision"); + throw std::runtime_error("Unexpected collision for file name hash " + + std::to_string(fileRecord->nameHash) + + " in set for folder name hash " + + std::to_string(folderHash)); } } } diff --git a/src/tests/api/internals/bsa_test.h b/src/tests/api/internals/bsa_test.h index 1b3a7bf3..3eee63fa 100644 --- a/src/tests/api/internals/bsa_test.h +++ b/src/tests/api/internals/bsa_test.h @@ -87,6 +87,46 @@ TEST(GetAssetsInBethesdaArchive, shouldThrowIfFileCannotBeOpened) { EXPECT_THROW(GetAssetsInBethesdaArchive(path), std::runtime_error); } +TEST(GetAssetsInBethesdaArchive, shouldSupportGeneralBA2s) { + const auto path = + std::filesystem::u8path("./Fallout 4/Data/Blank - Main.ba2"); + + const auto assets = GetAssetsInBethesdaArchive(path); + + size_t filesCount = 0; + for (const auto& folder : assets) { + filesCount += folder.second.size(); + } + + EXPECT_EQ(1, assets.size()); + EXPECT_EQ(1, filesCount); + + ASSERT_EQ(1, assets.count(0xFB6D522F)); + + EXPECT_EQ(1, assets.find(0xFB6D522F)->second.size()); + EXPECT_EQ(1, assets.find(0xFB6D522F)->second.count(0x747874CA042B67)); +} + +TEST(GetAssetsInBethesdaArchive, shouldSupportTextureBA2s) { + const auto path = + std::filesystem::u8path("./Fallout 4/Data/Blank - Textures.ba2"); + + const auto assets = GetAssetsInBethesdaArchive(path); + + size_t filesCount = 0; + for (const auto& folder : assets) { + filesCount += folder.second.size(); + } + + EXPECT_EQ(1, assets.size()); + EXPECT_EQ(1, filesCount); + + ASSERT_EQ(1, assets.count(0xFB6D522F)); + + EXPECT_EQ(1, assets.find(0xFB6D522F)->second.size()); + EXPECT_EQ(1, assets.find(0xFB6D522F)->second.count(0x736464FA093378)); +} + TEST(GetAssetsInBethesdaArchives, shouldSkipFilesThatCannotBeRead) { std::vector paths( {std::filesystem::u8path("invalid.bsa"), diff --git a/src/tests/api/internals/plugin_test.h b/src/tests/api/internals/plugin_test.h index b1e7e227..e3a12bf8 100644 --- a/src/tests/api/internals/plugin_test.h +++ b/src/tests/api/internals/plugin_test.h @@ -73,7 +73,15 @@ protected: // Copy across archive files. const auto blankMasterDependentArchive = "Blank - Master Dependent" + GetArchiveFileExtension(GetParam()); - if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + if (GetParam() == GameType::fo4 || GetParam() == GameType::fo4vr) { + copyPlugin("./Fallout 4/Data", "Blank - Main.ba2"); + copyPlugin("./Fallout 4/Data", "Blank - Textures.ba2"); + + std::filesystem::copy_file("./Fallout 4/Data/Blank - Main.ba2", + dataPath / blankMasterDependentArchive); + ASSERT_TRUE( + std::filesystem::exists(dataPath / blankMasterDependentArchive)); + } else if (GetParam() == GameType::tes3) { out.open(dataPath / blankArchive); out.close(); } else { @@ -104,7 +112,9 @@ protected: out.open(nonAsciiPrefixArchivePath); out.close(); - game_.GetCache().CacheArchivePaths({dataPath / blankArchive, + game_.GetCache().CacheArchivePaths({dataPath / "Blank - Main.ba2", + dataPath / "Blank - Textures.ba2", + dataPath / blankArchive, dataPath / blankMasterDependentArchive, dataPath / blankSuffixArchive, dataPath / nonAsciiArchivePath, @@ -588,8 +598,10 @@ TEST_P(PluginTest, Plugin(game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, false) .GetAssetCount(); - if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + if (GetParam() == GameType::tes3) { EXPECT_EQ(0, assetCount); + } else if (GetParam() == GameType::fo4) { + EXPECT_EQ(2, assetCount); } else { EXPECT_EQ(1, assetCount); } @@ -609,7 +621,7 @@ TEST_P(PluginTest, game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, false); OtherPluginType plugin2; - if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + if (GetParam() == GameType::tes3) { EXPECT_FALSE(plugin1.DoAssetsOverlap(plugin2)); } else { EXPECT_THROW(plugin1.DoAssetsOverlap(plugin2), std::invalid_argument); @@ -653,7 +665,7 @@ TEST_P(PluginTest, game_.DataPath() / blankMasterDependentEsp, false); - if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + if (GetParam() == GameType::tes3) { // Morrowind plugins can't load assets. EXPECT_FALSE(plugin1.DoAssetsOverlap(plugin2)); EXPECT_FALSE(plugin2.DoAssetsOverlap(plugin1));