loot/libloot
0
0
Fork 0
mirror of https://github.com/loot/libloot.git synced 2026-07-27 14:16:01 -07:00
Code Issues Packages Projects Releases Wiki Activity

Calculate 64-bit hashes for BA2 files

Browse Source 
BA2 files contain 32-bit hashes, which are not enough to reliably avoid
hash collisions (the worst case amongst Fallout4.esm's BA2 files has a
~ 84% chance of collision).

This instead calculates 64-bit folder and file hashes from the file
paths stored at the end of the BA2 files, which means that worst case
becomes a ~ 0.00000004% chance of collision.

Despite the much lower odds, I still see the same number of warnings
about a hash pair already existing when trying to merge hashes from
different BA2 files, so the same file paths must be present across
multiple BA2 archives. I wasn't really expecting any different though,
as I'd expect the official archives not to contain any hash collisions:
the game developers have control over the contents so can choose names
to avoid collisions that would presumably reduce performance if not
cause issues. That's not the case when working with an arbitrary
collection of mods though, so I think using 64-bit hashes is worthwhile.
This commit is contained in:
Oliver Hamlet
2022-12-31 18:55:44 +00:00
parent 38cecac07d
commit 934cda3c53
2 changed files with 78 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/bsa.cpp
+66 -137
View File
@@ -24,6 +24,7 @@
#include "api/bsa.h"
#include <boost/algorithm/string.hpp>
#include <fstream>
#include "api/bsa_detail.h"
@@ -120,159 +121,97 @@ struct Header {
uint64_t filePathsOffset{0};
};
struct GeneralFileRecord {
uint32_t fileHash{
0}; // Seems to be a hash of the file basename (without extension)
uint32_t extension{0}; // First 4 bytes of file extension.
uint32_t folderHash{0}; // Same value for files in the same directory.
uint32_t unknown1{0}; // Always 00 01 10 00?
uint32_t dataOffset{0}; // Offset from start of file.
uint32_t unknown2{0}; // Always null?
uint32_t unknown3{
0}; // Length of the compressed data block (or null if uncompressed).
uint32_t dataLength{0}; // Length of the uncompressed data block.
uint32_t unknown4{0}; // Always 0D F0 AD BA?
};
struct TextureFileRecord {
uint32_t fileHash{0};
uint32_t extension{0};
uint32_t folderHash{0};
uint8_t unknown1{0}; // Always null?
uint8_t subrecordCount{0};
uint16_t subrecordLength{0}; // Length of a single subrecord, always 24?
uint64_t unknown2{0};
};
void StoreHashes(std::map<uint64_t, std::set<uint64_t>>& folderFileHashes,
const uint32_t fileHash,
const uint32_t extension,
const uint32_t folderHash) {
// The file hash can't be used on its own as it's common for two files
// in the same directory to have the same basename but different
// extension, so treat the extension as the high bytes of a 64-bit hash.
const uint64_t fileAndExtensionHash =
uint64_t{fileHash} | (uint64_t{extension} << 32);
const auto folderResult = folderFileHashes.emplace(
folderHash, std::set<uint64_t>({fileAndExtensionHash}));
const uint64_t fileHash,
const uint64_t folderHash) {
const auto folderResult =
folderFileHashes.emplace(folderHash, std::set<uint64_t>({fileHash}));
if (!folderResult.second) {
// Folder hash already stored, add file hash to existing set.
const auto fileResult =
folderResult.first->second.insert(fileAndExtensionHash);
const auto fileResult = folderResult.first->second.insert(fileHash);
if (!fileResult.second) {
const auto message = fmt::format(
"Unexpected collision for file name hash {:x} in set for folder name "
"hash {:x}",
fileAndExtensionHash,
fileHash,
folderHash);
throw std::runtime_error(message);
}
}
}
std::map<uint64_t, std::set<uint64_t>> GetAssetsInGeneralBA2(
// Normalise the path the same way that BA2 hashes do (it's thet same as for
// BSAs).
void NormalisePath(std::string& filePath) {
for (size_t i = 0; i < filePath.size(); ++i) {
// Ignore any non-ASCII characters.
if (filePath[i] > 127) {
continue;
}
// Replace any forwardslashes with backslashes and
// lowercase any other characters.
filePath[i] = filePath[i] == '/'
? '\\'
: static_cast<char>(std::tolower(
static_cast<unsigned char>(filePath[i])));
}
}
std::map<uint64_t, std::set<uint64_t>> GetAssetsInBA2FromFilePaths(
std::istream& in,
const Header& header,
const std::filesystem::path& archivePath) {
const Header& header) {
// BA2s use 32-bit hashes and I've observed collisions between different
// official Fallout 4 BA2s, so calculate new 64-bit hashes instead of
// using the hashes in the BA2.
const auto logger = getLogger();
std::map<uint64_t, std::set<uint64_t>> folderFileHashes;
// Skip to list of file paths at the end of the BA2.
in.seekg(header.filePathsOffset, std::ios_base::beg);
// The file paths are prefixed by a two-byte length, and not null-terminated.
for (size_t i = 0; i < header.fileCount; ++i) {
GeneralFileRecord fileRecord;
in.read(reinterpret_cast<char*>(&fileRecord), sizeof(GeneralFileRecord));
uint16_t pathLength;
in.read(reinterpret_cast<char*>(&pathLength), sizeof(pathLength));
// Validate assumptions (don't error if invalid, because they don't
// actually matter for LOOT).
if (fileRecord.unknown1 != 0x100100) {
logger->warn(
"Unexpected value for unknown1 field in file record with hash {:x} "
"in BA2 file at \"{}\": {}",
fileRecord.fileHash,
archivePath.u8string(),
fileRecord.unknown1);
std::string filePath(pathLength, '\0');
in.read(filePath.data(), pathLength);
// Normalise the path the same as is done for BSA/BA2 hash calculation,
// so that equivalent but not equal paths (e.g. due to upper/lowercase
// differences) are hashed to the same value.
NormalisePath(filePath);
// Trim trailing and leading slashes.
boost::trim_if(filePath, [](const char c) { return c == '\\'; });
// Now split the path so that its folder and file hashes can be calculated.
const auto index = filePath.rfind("\\");
if (index == std::string::npos) {
// No slash, no directory, use a hash of zero.
const uint64_t fileHash = std::hash<std::string>{}(filePath);
const uint64_t folderHash = 0;
StoreHashes(folderFileHashes, fileHash, folderHash);
} else {
// Split the string in two.
const auto folderPath = filePath.substr(0, index);
filePath = filePath.substr(index + 1);
const uint64_t fileHash = std::hash<std::string>{}(filePath);
const uint64_t folderHash = std::hash<std::string>{}(folderPath);
StoreHashes(folderFileHashes, fileHash, folderHash);
}
if (fileRecord.unknown2 != 0) {
logger->warn(
"Unexpected value for unknown2 field in file record with hash {:x} "
"in BA2 file at \"{}\": {}",
fileRecord.fileHash,
archivePath.u8string(),
fileRecord.unknown2);
}
if (fileRecord.unknown4 != 0xBAADF00D) {
logger->warn(
"Unexpected value for unknown4 field in file record with hash {:x} "
"in BA2 file at \"{}\": {}",
fileRecord.fileHash,
archivePath.u8string(),
fileRecord.unknown4);
}
// Now store the hashes.
StoreHashes(folderFileHashes,
fileRecord.fileHash,
fileRecord.extension,
fileRecord.folderHash);
}
return folderFileHashes;
}
std::map<uint64_t, std::set<uint64_t>> GetAssetsInTextureBA2(
std::istream& in,
const Header& header,
const std::filesystem::path& archivePath) {
const auto logger = getLogger();
std::map<uint64_t, std::set<uint64_t>> folderFileHashes;
for (size_t i = 0; i < header.fileCount; ++i) {
TextureFileRecord fileRecord;
in.read(reinterpret_cast<char*>(&fileRecord), sizeof(TextureFileRecord));
// Skip over this file record's subrecords.
in.ignore(fileRecord.subrecordCount * fileRecord.subrecordLength);
// Validate assumptions (don't error if invalid, because they don't
// actually matter for LOOT).
if (fileRecord.unknown1 != 0) {
logger->warn(
"Unexpected value for unknown1 field in file record with hash {:x} "
"in BA2 file at \"{}\": {}",
fileRecord.fileHash,
archivePath.u8string(),
fileRecord.unknown1);
}
if (fileRecord.subrecordLength != 24) {
logger->warn(
"Unexpected value for subrecordLength field in file record with hash "
"{:x} in BA2 file at \"{}\": {}",
fileRecord.fileHash,
archivePath.u8string(),
fileRecord.subrecordLength);
}
// Now store the hashes.
StoreHashes(folderFileHashes,
fileRecord.fileHash,
fileRecord.extension,
fileRecord.folderHash);
}
return folderFileHashes;
}
std::map<uint64_t, std::set<uint64_t>> GetAssetsInBA2(
std::istream& in,
const Header& header,
const std::filesystem::path& archivePath) {
std::map<uint64_t, std::set<uint64_t>> GetAssetsInBA2(std::istream& in,
const Header& header) {
// Validate the header.
if (header.typeId != BA2_TYPE_ID) {
throw std::runtime_error("BA2 file header type ID is invalid");
@@ -287,17 +226,7 @@ std::map<uint64_t, std::set<uint64_t>> GetAssetsInBA2(
throw std::runtime_error("BA2 file header archive type is invalid");
}
// BA2s don't have the same structure as BSAs, for general BA2s everything is
// stored in a flatter structure, and for texture BA2s a more specialised
// structure is used. While there are still directory and file hashes, they're
// 32-bit, not 64-bit, which makes collisions more likely. Still, try to use
// them for simplicity.
if (header.archiveType == BA2_GENERAL_TYPE) {
return GetAssetsInGeneralBA2(in, header, archivePath);
}
return GetAssetsInTextureBA2(in, header, archivePath);
return GetAssetsInBA2FromFilePaths(in, header);
}
}
@@ -360,7 +289,7 @@ std::map<uint64_t, std::set<uint64_t>> GetAssetsInBethesdaArchive(
in.read(reinterpret_cast<char*>(&header) + typeId.size(),
sizeof(ba2::Header) - typeId.size());
return ba2::GetAssetsInBA2(in, header, archivePath);
return ba2::GetAssetsInBA2(in, header);
}
throw std::runtime_error("Bethesda archive has unrecognised typeId");
src/tests/api/internals/bsa_test.h
+12 -6
View File
@@ -90,6 +90,9 @@ TEST(GetAssetsInBethesdaArchive, shouldThrowIfFileCannotBeOpened) {
TEST(GetAssetsInBethesdaArchive, shouldSupportGeneralBA2s) {
const auto path =
std::filesystem::u8path("./Fallout 4/Data/Blank - Main.ba2");
const uint64_t folderHash =
std::hash<std::string>{}("dev\\git\\testing-plugins");
const uint64_t fileHash = std::hash<std::string>{}("license.txt");
const auto assets = GetAssetsInBethesdaArchive(path);
@@ -101,15 +104,18 @@ TEST(GetAssetsInBethesdaArchive, shouldSupportGeneralBA2s) {
EXPECT_EQ(1, assets.size());
EXPECT_EQ(1, filesCount);
ASSERT_EQ(1, assets.count(0xFB6D522F));
ASSERT_EQ(1, assets.count(folderHash));
EXPECT_EQ(1, assets.find(0xFB6D522F)->second.size());
EXPECT_EQ(1, assets.find(0xFB6D522F)->second.count(0x747874CA042B67));
EXPECT_EQ(1, assets.find(folderHash)->second.size());
EXPECT_EQ(1, assets.find(folderHash)->second.count(fileHash));
}
TEST(GetAssetsInBethesdaArchive, shouldSupportTextureBA2s) {
const auto path =
std::filesystem::u8path("./Fallout 4/Data/Blank - Textures.ba2");
const uint64_t folderHash =
std::hash<std::string>{}("dev\\git\\testing-plugins");
const uint64_t fileHash = std::hash<std::string>{}("blank.dds");
const auto assets = GetAssetsInBethesdaArchive(path);
@@ -121,10 +127,10 @@ TEST(GetAssetsInBethesdaArchive, shouldSupportTextureBA2s) {
EXPECT_EQ(1, assets.size());
EXPECT_EQ(1, filesCount);
ASSERT_EQ(1, assets.count(0xFB6D522F));
ASSERT_EQ(1, assets.count(folderHash));
EXPECT_EQ(1, assets.find(0xFB6D522F)->second.size());
EXPECT_EQ(1, assets.find(0xFB6D522F)->second.count(0x736464FA093378));
EXPECT_EQ(1, assets.find(folderHash)->second.size());
EXPECT_EQ(1, assets.find(folderHash)->second.count(fileHash));
}
TEST(GetAssetsInBethesdaArchives, shouldSkipFilesThatCannotBeRead) {