diff --git a/CMakeLists.txt b/CMakeLists.txt index dde651c5..93ab9b9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,8 +122,8 @@ endif() ExternalProject_Add(testing-plugins PREFIX "external" - URL "https://github.com/Ortham/testing-plugins/archive/1.4.1.tar.gz" - URL_HASH "SHA256=57f82495a737013c1b828d43fbe87edcb57768f85c6416e8d17f273d692691cf" + URL "https://github.com/Ortham/testing-plugins/archive/1.5.0.tar.gz" + URL_HASH "SHA256=98c9094fb0f0152b1af7a6206950207f7ddc6602cd44ed67ebf70603ef490791" CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "") diff --git a/docs/api/sorting.rst b/docs/api/sorting.rst index 02929cd2..6161c579 100644 --- a/docs/api/sorting.rst +++ b/docs/api/sorting.rst @@ -59,9 +59,9 @@ can be corrected. Plugin overlap edges are then added. Two plugins overlap if they contain the same record, i.e. if they both edit the same record or if one edits a record the -other plugin adds. Plugins also overlap if they both load one or more BSAs and -the BSAs loaded by one plugin contain data for a file path that is also included -in the BSAs loaded by the other plugin. +other plugin adds. Plugins also overlap if they both load one or more BSAs (BA2s +for Fallout 4) and the BSAs loaded by one plugin contain data for a file path +that is also included in the BSAs loaded by the other plugin. For each plugin, skip it if it overrides no records, otherwise iterate over all other plugins. diff --git a/src/api/bsa.cpp b/src/api/bsa.cpp index 1fe3005b..e18584af 100644 --- a/src/api/bsa.cpp +++ b/src/api/bsa.cpp @@ -39,28 +39,31 @@ BSA format documentation: */ constexpr std::array BSA_TYPE_ID = {'B', 'S', 'A', '\0'}; +constexpr std::array BA2_TYPE_ID = {'B', 'T', 'D', 'X'}; +constexpr std::array BA2_GENERAL_TYPE = {'G', 'N', 'R', 'L'}; +constexpr std::array BA2_TEXTURE_TYPE = {'D', 'X', '1', '0'}; -namespace bsa::v103 { +namespace bsa { +namespace v103 { struct FolderRecord { uint64_t nameHash{0}; uint32_t fileCount{0}; uint32_t fileRecordsOffset{0}; }; -std::map> GetAssetsInBethesdaArchive( - std::istream& in, - const Header& header) { - return detail::GetAssetsInBethesdaArchive(in, header); +std::map> GetAssetsInBSA(std::istream& in, + const Header& header) { + return detail::GetAssetsInBSA(in, header); } } -namespace bsa::v104 { -using bsa::v103::FolderRecord; +namespace v104 { +using v103::FolderRecord; -using v103::GetAssetsInBethesdaArchive; +using v103::GetAssetsInBSA; } -namespace bsa::v105 { +namespace v105 { struct FolderRecord { uint64_t nameHash{0}; uint32_t fileCount{0}; @@ -69,10 +72,232 @@ struct FolderRecord { uint32_t padding2{0}; }; -std::map> GetAssetsInBethesdaArchive( +std::map> GetAssetsInBSA(std::istream& in, + const Header& header) { + return detail::GetAssetsInBSA(in, header); +} +} + +std::map> GetAssetsInBSA( std::istream& in, - const Header& header) { - return detail::GetAssetsInBethesdaArchive(in, header); + const bsa::Header& header) { + const auto logger = getLogger(); + + // Validate the header. + if (header.typeId != BSA_TYPE_ID || + !(header.version == 103 || header.version == 104 || + header.version == 105) || + header.recordsOffset != 36) { + throw std::runtime_error("BSA file has an invalid header"); + } + + if ((header.archiveFlags & 0x40) != 0) { + throw std::runtime_error("BSA file uses big-endian numbers"); + } + + if (header.version == 103) { + return bsa::v103::GetAssetsInBSA(in, header); + } + + if (header.version == 104) { + return bsa::v104::GetAssetsInBSA(in, header); + } + + if (header.version == 105) { + return bsa::v105::GetAssetsInBSA(in, header); + } + + throw std::runtime_error("BSA file has an unrecognised version"); +} +} + +namespace ba2 { +struct Header { + std::array typeId; + uint32_t version{0}; + std::array archiveType; + uint32_t fileCount{0}; + uint64_t filePathsOffset{0}; +}; + +struct GeneralFileRecord { + uint32_t fileHash{ + 0}; // Seems to be a hash of the file basename (without extension) + uint32_t extension{0}; // First 4 bytes of file extension. + uint32_t folderHash{0}; // Same value for files in the same directory. + uint32_t unknown1{0}; // Always 00 01 10 00? + uint32_t dataOffset{0}; // Offset from start of file. + uint32_t unknown2{0}; // Always null? + uint32_t unknown3{ + 0}; // Length of the compressed data block (or null if uncompressed). + uint32_t dataLength{0}; // Length of the uncompressed data block. + uint32_t unknown4{0}; // Always 0D F0 AD BA? +}; + +struct TextureFileRecord { + uint32_t fileHash{0}; + uint32_t extension{0}; + uint32_t folderHash{0}; + uint8_t unknown1{0}; // Always null? + uint8_t subrecordCount{0}; + uint16_t subrecordLength{0}; // Length of a single subrecord, always 24? + uint64_t unknown2{0}; +}; + +void StoreHashes(std::map>& folderFileHashes, + const uint32_t fileHash, + const uint32_t extension, + const uint32_t folderHash) { + // The file hash can't be used on its own as it's common for two files + // in the same directory to have the same basename but different + // extension, so treat the extension as the high bytes of a 64-bit hash. + const uint64_t fileAndExtensionHash = + uint64_t{fileHash} | (uint64_t{extension} << 32); + + const auto folderResult = folderFileHashes.emplace( + folderHash, std::set({fileAndExtensionHash})); + + if (!folderResult.second) { + // Folder hash already stored, add file hash to existing set. + const auto fileResult = + folderResult.first->second.insert(fileAndExtensionHash); + + if (!fileResult.second) { + const auto message = fmt::format( + "Unexpected collision for file name hash {:x} in set for folder name " + "hash {:x}", + fileAndExtensionHash, + folderHash); + throw std::runtime_error(message); + } + } +} + +std::map> GetAssetsInGeneralBA2( + std::istream& in, + const Header& header, + const std::filesystem::path& archivePath) { + const auto logger = getLogger(); + + std::map> folderFileHashes; + + for (size_t i = 0; i < header.fileCount; ++i) { + GeneralFileRecord fileRecord; + in.read(reinterpret_cast(&fileRecord), sizeof(GeneralFileRecord)); + + // Validate assumptions (don't error if invalid, because they don't + // actually matter for LOOT). + if (fileRecord.unknown1 != 0x100100) { + logger->warn( + "Unexpected value for unknown1 field in file record with hash {:x} " + "in BA2 file at \"{}\": {}", + fileRecord.fileHash, + archivePath.u8string(), + fileRecord.unknown1); + } + + if (fileRecord.unknown2 != 0) { + logger->warn( + "Unexpected value for unknown2 field in file record with hash {:x} " + "in BA2 file at \"{}\": {}", + fileRecord.fileHash, + archivePath.u8string(), + fileRecord.unknown2); + } + + if (fileRecord.unknown4 != 0xBAADF00D) { + logger->warn( + "Unexpected value for unknown4 field in file record with hash {:x} " + "in BA2 file at \"{}\": {}", + fileRecord.fileHash, + archivePath.u8string(), + fileRecord.unknown4); + } + + // Now store the hashes. + StoreHashes(folderFileHashes, + fileRecord.fileHash, + fileRecord.extension, + fileRecord.folderHash); + } + + return folderFileHashes; +} + +std::map> GetAssetsInTextureBA2( + std::istream& in, + const Header& header, + const std::filesystem::path& archivePath) { + const auto logger = getLogger(); + + std::map> folderFileHashes; + + for (size_t i = 0; i < header.fileCount; ++i) { + TextureFileRecord fileRecord; + in.read(reinterpret_cast(&fileRecord), sizeof(TextureFileRecord)); + + // Skip over this file record's subrecords. + in.ignore(fileRecord.subrecordCount * fileRecord.subrecordLength); + + // Validate assumptions (don't error if invalid, because they don't + // actually matter for LOOT). + if (fileRecord.unknown1 != 0) { + logger->warn( + "Unexpected value for unknown1 field in file record with hash {:x} " + "in BA2 file at \"{}\": {}", + fileRecord.fileHash, + archivePath.u8string(), + fileRecord.unknown1); + } + + if (fileRecord.subrecordLength != 24) { + logger->warn( + "Unexpected value for subrecordLength field in file record with hash " + "{:x} in BA2 file at \"{}\": {}", + fileRecord.fileHash, + archivePath.u8string(), + fileRecord.subrecordLength); + } + + // Now store the hashes. + StoreHashes(folderFileHashes, + fileRecord.fileHash, + fileRecord.extension, + fileRecord.folderHash); + } + + return folderFileHashes; +} + +std::map> GetAssetsInBA2( + std::istream& in, + const Header& header, + const std::filesystem::path& archivePath) { + // Validate the header. + if (header.typeId != BA2_TYPE_ID) { + throw std::runtime_error("BA2 file header type ID is invalid"); + } + + if (header.version != 1) { + throw std::runtime_error("BA2 file header version is invalid"); + } + + if (header.archiveType != BA2_GENERAL_TYPE && + header.archiveType != BA2_TEXTURE_TYPE) { + throw std::runtime_error("BA2 file header archive type is invalid"); + } + + // BA2s don't have the same structure as BSAs, for general BA2s everything is + // stored in a flatter structure, and for texture BA2s a more specialised + // structure is used. While there are still directory and file hashes, they're + // 32-bit, not 64-bit, which makes collisions more likely. Still, try to use + // them for simplicity. + + if (header.archiveType == BA2_GENERAL_TYPE) { + return GetAssetsInGeneralBA2(in, header, archivePath); + } + + return GetAssetsInTextureBA2(in, header, archivePath); } } @@ -104,8 +329,6 @@ std::map> GetAssetsInBethesdaArchive( if (!std::filesystem::exists(archivePath)) { if (logger) { - logger->error("Bethesda archive path \"{}\" does not exist!", - archivePath.u8string()); throw std::runtime_error("Bethesda archive does not exist"); } } @@ -115,47 +338,32 @@ std::map> GetAssetsInBethesdaArchive( std::ios::eofbit); // Causes ifstream::failure to be thrown if // a problem is encountered. - bsa::Header header; - in.read(reinterpret_cast(&header), sizeof(bsa::Header)); + std::array typeId; + in.read(typeId.data(), typeId.size()); - // Validate the header. - if (header.typeId != BSA_TYPE_ID || - !(header.version == 103 || header.version == 104 || - header.version == 105) || - header.recordsOffset != 36) { - if (logger) { - logger->error("Bethesda archive at \"{}\" has an invalid header.", - archivePath.u8string()); - throw std::runtime_error("Bethesda archive has an invalid header"); - } + if (typeId == BSA_TYPE_ID) { + bsa::Header header; + header.typeId = typeId; + + // Read the rest of the header. + in.read(reinterpret_cast(&header) + typeId.size(), + sizeof(bsa::Header) - typeId.size()); + + return bsa::GetAssetsInBSA(in, header); } - if ((header.archiveFlags & 0x40) != 0) { - if (logger) { - logger->error("BSA file at \"{}\" uses big-endian numbers."); - } - throw std::runtime_error("BSA file uses big-endian numbers"); + if (typeId == BA2_TYPE_ID) { + ba2::Header header; + header.typeId = typeId; + + // Read the rest of the header. + in.read(reinterpret_cast(&header) + typeId.size(), + sizeof(ba2::Header) - typeId.size()); + + return ba2::GetAssetsInBA2(in, header, archivePath); } - if (header.version == 103) { - return bsa::v103::GetAssetsInBethesdaArchive(in, header); - } - - if (header.version == 104) { - return bsa::v104::GetAssetsInBethesdaArchive(in, header); - } - - if (header.version == 105) { - return bsa::v105::GetAssetsInBethesdaArchive(in, header); - } - - if (logger) { - logger->error("Unrecognised BSA version {} in archive at \"{}\"", - header.version, - archivePath.u8string()); - } - - throw std::runtime_error("BSA file has an unrecognised version"); + throw std::runtime_error("Bethesda archive has unrecognised typeId"); } std::map> GetAssetsInBethesdaArchives( diff --git a/src/api/bsa_detail.h b/src/api/bsa_detail.h index a851a17f..271f7d06 100644 --- a/src/api/bsa_detail.h +++ b/src/api/bsa_detail.h @@ -56,9 +56,8 @@ struct FileRecord { namespace loot::bsa::detail { template -std::map> GetAssetsInBethesdaArchive( - std::istream& in, - const Header& header) { +std::map> GetAssetsInBSA(std::istream& in, + const Header& header) { const auto logger = getLogger(); std::vector folderRecords(header.folderCount); @@ -93,10 +92,8 @@ std::map> GetAssetsInBethesdaArchive( folderFileHashes.emplace(folderHash, std::set()); if (!folderResult.second) { - if (logger) { - logger->warn("Folder name hash {} is already in map", folderHash); - } - throw std::runtime_error("Unexpected folder name hash collision"); + throw std::runtime_error("Unexpected collision for folder name hash " + + std::to_string(folderHash)); } size_t fileRecordsOffset = 0; @@ -126,13 +123,10 @@ std::map> GetAssetsInBethesdaArchive( folderResult.first->second.insert(fileRecord->nameHash); if (!result.second) { - if (logger) { - logger->warn( - "File name hash {} is already in the set for folder name hash {}", - fileRecord->nameHash, - folderHash); - } - throw std::runtime_error("Unexpected file name hash collision"); + throw std::runtime_error("Unexpected collision for file name hash " + + std::to_string(fileRecord->nameHash) + + " in set for folder name hash " + + std::to_string(folderHash)); } } } diff --git a/src/tests/api/internals/bsa_test.h b/src/tests/api/internals/bsa_test.h index 1b3a7bf3..3eee63fa 100644 --- a/src/tests/api/internals/bsa_test.h +++ b/src/tests/api/internals/bsa_test.h @@ -87,6 +87,46 @@ TEST(GetAssetsInBethesdaArchive, shouldThrowIfFileCannotBeOpened) { EXPECT_THROW(GetAssetsInBethesdaArchive(path), std::runtime_error); } +TEST(GetAssetsInBethesdaArchive, shouldSupportGeneralBA2s) { + const auto path = + std::filesystem::u8path("./Fallout 4/Data/Blank - Main.ba2"); + + const auto assets = GetAssetsInBethesdaArchive(path); + + size_t filesCount = 0; + for (const auto& folder : assets) { + filesCount += folder.second.size(); + } + + EXPECT_EQ(1, assets.size()); + EXPECT_EQ(1, filesCount); + + ASSERT_EQ(1, assets.count(0xFB6D522F)); + + EXPECT_EQ(1, assets.find(0xFB6D522F)->second.size()); + EXPECT_EQ(1, assets.find(0xFB6D522F)->second.count(0x747874CA042B67)); +} + +TEST(GetAssetsInBethesdaArchive, shouldSupportTextureBA2s) { + const auto path = + std::filesystem::u8path("./Fallout 4/Data/Blank - Textures.ba2"); + + const auto assets = GetAssetsInBethesdaArchive(path); + + size_t filesCount = 0; + for (const auto& folder : assets) { + filesCount += folder.second.size(); + } + + EXPECT_EQ(1, assets.size()); + EXPECT_EQ(1, filesCount); + + ASSERT_EQ(1, assets.count(0xFB6D522F)); + + EXPECT_EQ(1, assets.find(0xFB6D522F)->second.size()); + EXPECT_EQ(1, assets.find(0xFB6D522F)->second.count(0x736464FA093378)); +} + TEST(GetAssetsInBethesdaArchives, shouldSkipFilesThatCannotBeRead) { std::vector paths( {std::filesystem::u8path("invalid.bsa"), diff --git a/src/tests/api/internals/plugin_test.h b/src/tests/api/internals/plugin_test.h index b1e7e227..e3a12bf8 100644 --- a/src/tests/api/internals/plugin_test.h +++ b/src/tests/api/internals/plugin_test.h @@ -73,7 +73,15 @@ protected: // Copy across archive files. const auto blankMasterDependentArchive = "Blank - Master Dependent" + GetArchiveFileExtension(GetParam()); - if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + if (GetParam() == GameType::fo4 || GetParam() == GameType::fo4vr) { + copyPlugin("./Fallout 4/Data", "Blank - Main.ba2"); + copyPlugin("./Fallout 4/Data", "Blank - Textures.ba2"); + + std::filesystem::copy_file("./Fallout 4/Data/Blank - Main.ba2", + dataPath / blankMasterDependentArchive); + ASSERT_TRUE( + std::filesystem::exists(dataPath / blankMasterDependentArchive)); + } else if (GetParam() == GameType::tes3) { out.open(dataPath / blankArchive); out.close(); } else { @@ -104,7 +112,9 @@ protected: out.open(nonAsciiPrefixArchivePath); out.close(); - game_.GetCache().CacheArchivePaths({dataPath / blankArchive, + game_.GetCache().CacheArchivePaths({dataPath / "Blank - Main.ba2", + dataPath / "Blank - Textures.ba2", + dataPath / blankArchive, dataPath / blankMasterDependentArchive, dataPath / blankSuffixArchive, dataPath / nonAsciiArchivePath, @@ -588,8 +598,10 @@ TEST_P(PluginTest, Plugin(game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, false) .GetAssetCount(); - if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + if (GetParam() == GameType::tes3) { EXPECT_EQ(0, assetCount); + } else if (GetParam() == GameType::fo4) { + EXPECT_EQ(2, assetCount); } else { EXPECT_EQ(1, assetCount); } @@ -609,7 +621,7 @@ TEST_P(PluginTest, game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, false); OtherPluginType plugin2; - if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + if (GetParam() == GameType::tes3) { EXPECT_FALSE(plugin1.DoAssetsOverlap(plugin2)); } else { EXPECT_THROW(plugin1.DoAssetsOverlap(plugin2), std::invalid_argument); @@ -653,7 +665,7 @@ TEST_P(PluginTest, game_.DataPath() / blankMasterDependentEsp, false); - if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + if (GetParam() == GameType::tes3) { // Morrowind plugins can't load assets. EXPECT_FALSE(plugin1.DoAssetsOverlap(plugin2)); EXPECT_FALSE(plugin2.DoAssetsOverlap(plugin1));