Add support for checking assets in BA2 files

They're used by Fallout 4.

BA2 file and folder hashes are 32-bit, so collisions are much more
likely than with BSAs, which use 64-bit hashes. I put in some logging
to check the likelihood of collisions, and found that the largest
number of assets loaded was by Fallout4.esm (no surprise), which loaded
371182 files across 5746 folders, with the most files in one folder
being 124871. That puts the probability of a hash collision within that
folder at above 80%.

I did see different Fallout4 -*.ba2 files contain files with the same
combination of folder and file hashes, and similar for
DLCUltraHighResolution -*.ba2 files, so I'm going to try calculating
64-bit hashes from the file paths stored in the BA2 files.

CMakeLists.txt

@@ -122,8 +122,8 @@ endif()
 
 ExternalProject_Add(testing-plugins
     PREFIX "external"
-    URL "https://github.com/Ortham/testing-plugins/archive/1.4.1.tar.gz"
-    URL_HASH "SHA256=57f82495a737013c1b828d43fbe87edcb57768f85c6416e8d17f273d692691cf"
+    URL "https://github.com/Ortham/testing-plugins/archive/1.5.0.tar.gz"
+    URL_HASH "SHA256=98c9094fb0f0152b1af7a6206950207f7ddc6602cd44ed67ebf70603ef490791"
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
     INSTALL_COMMAND "")

docs/api/sorting.rst

@@ -59,9 +59,9 @@ can be corrected.
 
 Plugin overlap edges are then added. Two plugins overlap if they contain the
 same record, i.e. if they both edit the same record or if one edits a record the
-other plugin adds. Plugins also overlap if they both load one or more BSAs and
-the BSAs loaded by one plugin contain data for a file path that is also included
-in the BSAs loaded by the other plugin.
+other plugin adds. Plugins also overlap if they both load one or more BSAs (BA2s
+for Fallout 4) and the BSAs loaded by one plugin contain data for a file path
+that is also included in the BSAs loaded by the other plugin.
 
 For each plugin, skip it if it overrides no records, otherwise iterate over all
 other plugins.

src/api/bsa.cpp

@@ -39,28 +39,31 @@ BSA format documentation:
 
 */
 constexpr std::array<char, 4> BSA_TYPE_ID = {'B', 'S', 'A', '\0'};
+constexpr std::array<char, 4> BA2_TYPE_ID = {'B', 'T', 'D', 'X'};
+constexpr std::array<char, 4> BA2_GENERAL_TYPE = {'G', 'N', 'R', 'L'};
+constexpr std::array<char, 4> BA2_TEXTURE_TYPE = {'D', 'X', '1', '0'};
 
-namespace bsa::v103 {
+namespace bsa {
+namespace v103 {
 struct FolderRecord {
   uint64_t nameHash{0};
   uint32_t fileCount{0};
   uint32_t fileRecordsOffset{0};
 };
 
-std::map<uint64_t, std::set<uint64_t>> GetAssetsInBethesdaArchive(
-    std::istream& in,
-    const Header& header) {
-  return detail::GetAssetsInBethesdaArchive<FolderRecord>(in, header);
+std::map<uint64_t, std::set<uint64_t>> GetAssetsInBSA(std::istream& in,
+                                                      const Header& header) {
+  return detail::GetAssetsInBSA<FolderRecord>(in, header);
 }
 }
 
-namespace bsa::v104 {
-using bsa::v103::FolderRecord;
+namespace v104 {
+using v103::FolderRecord;
 
-using v103::GetAssetsInBethesdaArchive;
+using v103::GetAssetsInBSA;
 }
 
-namespace bsa::v105 {
+namespace v105 {
 struct FolderRecord {
   uint64_t nameHash{0};
   uint32_t fileCount{0};
@@ -69,10 +72,232 @@ struct FolderRecord {
   uint32_t padding2{0};
 };
 
-std::map<uint64_t, std::set<uint64_t>> GetAssetsInBethesdaArchive(
+std::map<uint64_t, std::set<uint64_t>> GetAssetsInBSA(std::istream& in,
+                                                      const Header& header) {
+  return detail::GetAssetsInBSA<FolderRecord>(in, header);
+}
+}
+
+std::map<uint64_t, std::set<uint64_t>> GetAssetsInBSA(
     std::istream& in,
-    const Header& header) {
-  return detail::GetAssetsInBethesdaArchive<FolderRecord>(in, header);
+    const bsa::Header& header) {
+  const auto logger = getLogger();
+
+  // Validate the header.
+  if (header.typeId != BSA_TYPE_ID ||
+      !(header.version == 103 || header.version == 104 ||
+        header.version == 105) ||
+      header.recordsOffset != 36) {
+    throw std::runtime_error("BSA file has an invalid header");
+  }
+
+  if ((header.archiveFlags & 0x40) != 0) {
+    throw std::runtime_error("BSA file uses big-endian numbers");
+  }
+
+  if (header.version == 103) {
+    return bsa::v103::GetAssetsInBSA(in, header);
+  }
+
+  if (header.version == 104) {
+    return bsa::v104::GetAssetsInBSA(in, header);
+  }
+
+  if (header.version == 105) {
+    return bsa::v105::GetAssetsInBSA(in, header);
+  }
+
+  throw std::runtime_error("BSA file has an unrecognised version");
+}
+}
+
+namespace ba2 {
+struct Header {
+  std::array<char, 4> typeId;
+  uint32_t version{0};
+  std::array<char, 4> archiveType;
+  uint32_t fileCount{0};
+  uint64_t filePathsOffset{0};
+};
+
+struct GeneralFileRecord {
+  uint32_t fileHash{
+      0};  // Seems to be a hash of the file basename (without extension)
+  uint32_t extension{0};   // First 4 bytes of file extension.
+  uint32_t folderHash{0};  // Same value for files in the same directory.
+  uint32_t unknown1{0};    // Always 00 01 10 00?
+  uint32_t dataOffset{0};  // Offset from start of file.
+  uint32_t unknown2{0};    // Always null?
+  uint32_t unknown3{
+      0};  // Length of the compressed data block (or null if uncompressed).
+  uint32_t dataLength{0};  // Length of the uncompressed data block.
+  uint32_t unknown4{0};    // Always 0D F0 AD BA?
+};
+
+struct TextureFileRecord {
+  uint32_t fileHash{0};
+  uint32_t extension{0};
+  uint32_t folderHash{0};
+  uint8_t unknown1{0};  // Always null?
+  uint8_t subrecordCount{0};
+  uint16_t subrecordLength{0};  // Length of a single subrecord, always 24?
+  uint64_t unknown2{0};
+};
+
+void StoreHashes(std::map<uint64_t, std::set<uint64_t>>& folderFileHashes,
+                 const uint32_t fileHash,
+                 const uint32_t extension,
+                 const uint32_t folderHash) {
+  // The file hash can't be used on its own as it's common for two files
+  // in the same directory to have the same basename but different
+  // extension, so treat the extension as the high bytes of a 64-bit hash.
+  const uint64_t fileAndExtensionHash =
+      uint64_t{fileHash} | (uint64_t{extension} << 32);
+
+  const auto folderResult = folderFileHashes.emplace(
+      folderHash, std::set<uint64_t>({fileAndExtensionHash}));
+
+  if (!folderResult.second) {
+    // Folder hash already stored, add file hash to existing set.
+    const auto fileResult =
+        folderResult.first->second.insert(fileAndExtensionHash);
+
+    if (!fileResult.second) {
+      const auto message = fmt::format(
+          "Unexpected collision for file name hash {:x} in set for folder name "
+          "hash {:x}",
+          fileAndExtensionHash,
+          folderHash);
+      throw std::runtime_error(message);
+    }
+  }
+}
+
+std::map<uint64_t, std::set<uint64_t>> GetAssetsInGeneralBA2(
+    std::istream& in,
+    const Header& header,
+    const std::filesystem::path& archivePath) {
+  const auto logger = getLogger();
+
+  std::map<uint64_t, std::set<uint64_t>> folderFileHashes;
+
+  for (size_t i = 0; i < header.fileCount; ++i) {
+    GeneralFileRecord fileRecord;
+    in.read(reinterpret_cast<char*>(&fileRecord), sizeof(GeneralFileRecord));
+
+    // Validate assumptions (don't error if invalid, because they don't
+    // actually matter for LOOT).
+    if (fileRecord.unknown1 != 0x100100) {
+      logger->warn(
+          "Unexpected value for unknown1 field in file record with hash {:x} "
+          "in BA2 file at \"{}\": {}",
+          fileRecord.fileHash,
+          archivePath.u8string(),
+          fileRecord.unknown1);
+    }
+
+    if (fileRecord.unknown2 != 0) {
+      logger->warn(
+          "Unexpected value for unknown2 field in file record with hash {:x} "
+          "in BA2 file at \"{}\": {}",
+          fileRecord.fileHash,
+          archivePath.u8string(),
+          fileRecord.unknown2);
+    }
+
+    if (fileRecord.unknown4 != 0xBAADF00D) {
+      logger->warn(
+          "Unexpected value for unknown4 field in file record with hash {:x} "
+          "in BA2 file at \"{}\": {}",
+          fileRecord.fileHash,
+          archivePath.u8string(),
+          fileRecord.unknown4);
+    }
+
+    // Now store the hashes.
+    StoreHashes(folderFileHashes,
+                fileRecord.fileHash,
+                fileRecord.extension,
+                fileRecord.folderHash);
+  }
+
+  return folderFileHashes;
+}
+
+std::map<uint64_t, std::set<uint64_t>> GetAssetsInTextureBA2(
+    std::istream& in,
+    const Header& header,
+    const std::filesystem::path& archivePath) {
+  const auto logger = getLogger();
+
+  std::map<uint64_t, std::set<uint64_t>> folderFileHashes;
+
+  for (size_t i = 0; i < header.fileCount; ++i) {
+    TextureFileRecord fileRecord;
+    in.read(reinterpret_cast<char*>(&fileRecord), sizeof(TextureFileRecord));
+
+    // Skip over this file record's subrecords.
+    in.ignore(fileRecord.subrecordCount * fileRecord.subrecordLength);
+
+    // Validate assumptions (don't error if invalid, because they don't
+    // actually matter for LOOT).
+    if (fileRecord.unknown1 != 0) {
+      logger->warn(
+          "Unexpected value for unknown1 field in file record with hash {:x} "
+          "in BA2 file at \"{}\": {}",
+          fileRecord.fileHash,
+          archivePath.u8string(),
+          fileRecord.unknown1);
+    }
+
+    if (fileRecord.subrecordLength != 24) {
+      logger->warn(
+          "Unexpected value for subrecordLength field in file record with hash "
+          "{:x} in BA2 file at \"{}\": {}",
+          fileRecord.fileHash,
+          archivePath.u8string(),
+          fileRecord.subrecordLength);
+    }
+
+    // Now store the hashes.
+    StoreHashes(folderFileHashes,
+                fileRecord.fileHash,
+                fileRecord.extension,
+                fileRecord.folderHash);
+  }
+
+  return folderFileHashes;
+}
+
+std::map<uint64_t, std::set<uint64_t>> GetAssetsInBA2(
+    std::istream& in,
+    const Header& header,
+    const std::filesystem::path& archivePath) {
+  // Validate the header.
+  if (header.typeId != BA2_TYPE_ID) {
+    throw std::runtime_error("BA2 file header type ID is invalid");
+  }
+
+  if (header.version != 1) {
+    throw std::runtime_error("BA2 file header version is invalid");
+  }
+
+  if (header.archiveType != BA2_GENERAL_TYPE &&
+      header.archiveType != BA2_TEXTURE_TYPE) {
+    throw std::runtime_error("BA2 file header archive type is invalid");
+  }
+
+  // BA2s don't have the same structure as BSAs, for general BA2s everything is
+  // stored in a flatter structure, and for texture BA2s a more specialised
+  // structure is used. While there are still directory and file hashes, they're
+  // 32-bit, not 64-bit, which makes collisions more likely. Still, try to use
+  // them for simplicity.
+
+  if (header.archiveType == BA2_GENERAL_TYPE) {
+    return GetAssetsInGeneralBA2(in, header, archivePath);
+  }
+
+  return GetAssetsInTextureBA2(in, header, archivePath);
 }
 }
 
@@ -104,8 +329,6 @@ std::map<uint64_t, std::set<uint64_t>> GetAssetsInBethesdaArchive(
 
   if (!std::filesystem::exists(archivePath)) {
     if (logger) {
-      logger->error("Bethesda archive path \"{}\" does not exist!",
-                    archivePath.u8string());
       throw std::runtime_error("Bethesda archive does not exist");
     }
   }
@@ -115,47 +338,32 @@ std::map<uint64_t, std::set<uint64_t>> GetAssetsInBethesdaArchive(
                 std::ios::eofbit);  // Causes ifstream::failure to be thrown if
                                     // a problem is encountered.
 
-  bsa::Header header;
-  in.read(reinterpret_cast<char*>(&header), sizeof(bsa::Header));
+  std::array<char, 4> typeId;
+  in.read(typeId.data(), typeId.size());
 
-  // Validate the header.
-  if (header.typeId != BSA_TYPE_ID ||
-      !(header.version == 103 || header.version == 104 ||
-        header.version == 105) ||
-      header.recordsOffset != 36) {
-    if (logger) {
-      logger->error("Bethesda archive at \"{}\" has an invalid header.",
-                    archivePath.u8string());
-      throw std::runtime_error("Bethesda archive has an invalid header");
-    }
+  if (typeId == BSA_TYPE_ID) {
+    bsa::Header header;
+    header.typeId = typeId;
+
+    // Read the rest of the header.
+    in.read(reinterpret_cast<char*>(&header) + typeId.size(),
+            sizeof(bsa::Header) - typeId.size());
+
+    return bsa::GetAssetsInBSA(in, header);
   }
 
-  if ((header.archiveFlags & 0x40) != 0) {
-    if (logger) {
-      logger->error("BSA file at \"{}\" uses big-endian numbers.");
-    }
-    throw std::runtime_error("BSA file uses big-endian numbers");
+  if (typeId == BA2_TYPE_ID) {
+    ba2::Header header;
+    header.typeId = typeId;
+
+    // Read the rest of the header.
+    in.read(reinterpret_cast<char*>(&header) + typeId.size(),
+            sizeof(ba2::Header) - typeId.size());
+
+    return ba2::GetAssetsInBA2(in, header, archivePath);
   }
 
-  if (header.version == 103) {
-    return bsa::v103::GetAssetsInBethesdaArchive(in, header);
-  }
-
-  if (header.version == 104) {
-    return bsa::v104::GetAssetsInBethesdaArchive(in, header);
-  }
-
-  if (header.version == 105) {
-    return bsa::v105::GetAssetsInBethesdaArchive(in, header);
-  }
-
-  if (logger) {
-    logger->error("Unrecognised BSA version {} in archive at \"{}\"",
-                  header.version,
-                  archivePath.u8string());
-  }
-
-  throw std::runtime_error("BSA file has an unrecognised version");
+  throw std::runtime_error("Bethesda archive has unrecognised typeId");
 }
 
 std::map<uint64_t, std::set<uint64_t>> GetAssetsInBethesdaArchives(

src/api/bsa_detail.h

@@ -56,9 +56,8 @@ struct FileRecord {
 
 namespace loot::bsa::detail {
 template<typename FolderRecord>
-std::map<uint64_t, std::set<uint64_t>> GetAssetsInBethesdaArchive(
-    std::istream& in,
-    const Header& header) {
+std::map<uint64_t, std::set<uint64_t>> GetAssetsInBSA(std::istream& in,
+                                                      const Header& header) {
   const auto logger = getLogger();
 
   std::vector<FolderRecord> folderRecords(header.folderCount);
@@ -93,10 +92,8 @@ std::map<uint64_t, std::set<uint64_t>> GetAssetsInBethesdaArchive(
         folderFileHashes.emplace(folderHash, std::set<uint64_t>());
 
     if (!folderResult.second) {
-      if (logger) {
-        logger->warn("Folder name hash {} is already in map", folderHash);
-      }
-      throw std::runtime_error("Unexpected folder name hash collision");
+      throw std::runtime_error("Unexpected collision for folder name hash " +
+                               std::to_string(folderHash));
     }
 
     size_t fileRecordsOffset = 0;
@@ -126,13 +123,10 @@ std::map<uint64_t, std::set<uint64_t>> GetAssetsInBethesdaArchive(
           folderResult.first->second.insert(fileRecord->nameHash);
 
       if (!result.second) {
-        if (logger) {
-          logger->warn(
-              "File name hash {} is already in the set for folder name hash {}",
-              fileRecord->nameHash,
-              folderHash);
-        }
-        throw std::runtime_error("Unexpected file name hash collision");
+        throw std::runtime_error("Unexpected collision for file name hash " +
+                                 std::to_string(fileRecord->nameHash) +
+                                 " in set for folder name hash " +
+                                 std::to_string(folderHash));
       }
     }
   }

src/tests/api/internals/bsa_test.h

@@ -87,6 +87,46 @@ TEST(GetAssetsInBethesdaArchive, shouldThrowIfFileCannotBeOpened) {
   EXPECT_THROW(GetAssetsInBethesdaArchive(path), std::runtime_error);
 }
 
+TEST(GetAssetsInBethesdaArchive, shouldSupportGeneralBA2s) {
+  const auto path =
+      std::filesystem::u8path("./Fallout 4/Data/Blank - Main.ba2");
+
+  const auto assets = GetAssetsInBethesdaArchive(path);
+
+  size_t filesCount = 0;
+  for (const auto& folder : assets) {
+    filesCount += folder.second.size();
+  }
+
+  EXPECT_EQ(1, assets.size());
+  EXPECT_EQ(1, filesCount);
+
+  ASSERT_EQ(1, assets.count(0xFB6D522F));
+
+  EXPECT_EQ(1, assets.find(0xFB6D522F)->second.size());
+  EXPECT_EQ(1, assets.find(0xFB6D522F)->second.count(0x747874CA042B67));
+}
+
+TEST(GetAssetsInBethesdaArchive, shouldSupportTextureBA2s) {
+  const auto path =
+      std::filesystem::u8path("./Fallout 4/Data/Blank - Textures.ba2");
+
+  const auto assets = GetAssetsInBethesdaArchive(path);
+
+  size_t filesCount = 0;
+  for (const auto& folder : assets) {
+    filesCount += folder.second.size();
+  }
+
+  EXPECT_EQ(1, assets.size());
+  EXPECT_EQ(1, filesCount);
+
+  ASSERT_EQ(1, assets.count(0xFB6D522F));
+
+  EXPECT_EQ(1, assets.find(0xFB6D522F)->second.size());
+  EXPECT_EQ(1, assets.find(0xFB6D522F)->second.count(0x736464FA093378));
+}
+
 TEST(GetAssetsInBethesdaArchives, shouldSkipFilesThatCannotBeRead) {
   std::vector<std::filesystem::path> paths(
       {std::filesystem::u8path("invalid.bsa"),

src/tests/api/internals/plugin_test.h

@@ -73,7 +73,15 @@ protected:
     // Copy across archive files.
     const auto blankMasterDependentArchive =
         "Blank - Master Dependent" + GetArchiveFileExtension(GetParam());
-    if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) {
+    if (GetParam() == GameType::fo4 || GetParam() == GameType::fo4vr) {
+      copyPlugin("./Fallout 4/Data", "Blank - Main.ba2");
+      copyPlugin("./Fallout 4/Data", "Blank - Textures.ba2");
+
+      std::filesystem::copy_file("./Fallout 4/Data/Blank - Main.ba2",
+                                 dataPath / blankMasterDependentArchive);
+      ASSERT_TRUE(
+          std::filesystem::exists(dataPath / blankMasterDependentArchive));
+    } else if (GetParam() == GameType::tes3) {
       out.open(dataPath / blankArchive);
       out.close();
     } else {
@@ -104,7 +112,9 @@ protected:
     out.open(nonAsciiPrefixArchivePath);
     out.close();
 
-    game_.GetCache().CacheArchivePaths({dataPath / blankArchive,
+    game_.GetCache().CacheArchivePaths({dataPath / "Blank - Main.ba2",
+                                        dataPath / "Blank - Textures.ba2",
+                                        dataPath / blankArchive,
                                         dataPath / blankMasterDependentArchive,
                                         dataPath / blankSuffixArchive,
                                         dataPath / nonAsciiArchivePath,
@@ -588,8 +598,10 @@ TEST_P(PluginTest,
       Plugin(game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, false)
           .GetAssetCount();
 
-  if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) {
+  if (GetParam() == GameType::tes3) {
     EXPECT_EQ(0, assetCount);
+  } else if (GetParam() == GameType::fo4) {
+    EXPECT_EQ(2, assetCount);
   } else {
     EXPECT_EQ(1, assetCount);
   }
@@ -609,7 +621,7 @@ TEST_P(PluginTest,
       game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, false);
   OtherPluginType plugin2;
 
-  if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) {
+  if (GetParam() == GameType::tes3) {
     EXPECT_FALSE(plugin1.DoAssetsOverlap(plugin2));
   } else {
     EXPECT_THROW(plugin1.DoAssetsOverlap(plugin2), std::invalid_argument);
@@ -653,7 +665,7 @@ TEST_P(PluginTest,
                  game_.DataPath() / blankMasterDependentEsp,
                  false);
 
-  if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) {
+  if (GetParam() == GameType::tes3) {
     // Morrowind plugins can't load assets.
     EXPECT_FALSE(plugin1.DoAssetsOverlap(plugin2));
     EXPECT_FALSE(plugin2.DoAssetsOverlap(plugin1));