From d05685aea44b09763a3dfa2f508f355311f3699a Mon Sep 17 00:00:00 2001 From: Oliver Hamlet Date: Thu, 15 Dec 2022 22:28:18 +0000 Subject: [PATCH] Take loaded BSAs into account during sorting This supports BSAs used from: * Oblivion * Fallout 3 * Fallout: New Vegas * Skyrim * Skyrim: Special Edition If Skyrim VR uses the same BSA format as Skyrim SE, that's also supported. Morrowind BSAs are intentionally not supported because they cannot be loaded by plugins and so are of no interest to LOOT. The BSA parsing code has been adapted from my abandoned libbsa library. This doesn't bother reading folder and file names as hash collisions seem pretty unlikely (2^64 possible folder hashes, and 2^64 possible file hashes per folder). The code checks and requires that BSAs use little-endian numbers, as while big-endian BSAs are apparently possible I've never seen one, and I don't want to try adding support without one to test against. --- CMakeLists.txt | 3 + docs/api/sorting.rst | 16 +- src/api/bsa.cpp | 236 ++++++++++++ src/api/bsa.h | 44 +++ src/api/bsa_detail.h | 144 ++++++++ src/api/plugin.cpp | 242 +++++++----- src/api/plugin.h | 13 +- src/api/sorting/plugin_graph.cpp | 61 ++- src/api/sorting/plugin_sorting_data.cpp | 9 + src/api/sorting/plugin_sorting_data.h | 3 + src/tests/api/internals/bsa_test.h | 157 ++++++++ src/tests/api/internals/main.cpp | 1 + src/tests/api/internals/plugin_test.h | 107 +++++- .../api/internals/sorting/plugin_graph_test.h | 346 +++++++++++++++++- src/tests/common_game_test_fixture.h | 24 +- 15 files changed, 1274 insertions(+), 132 deletions(-) create mode 100644 src/api/bsa.cpp create mode 100644 src/api/bsa.h create mode 100644 src/api/bsa_detail.h create mode 100644 src/tests/api/internals/bsa_test.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c8d0398..dde651c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -170,6 +170,7 @@ set(YAML_CPP_LIBRARIES "${BINARY_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY set(LIBLOOT_SRC_API_CPP_FILES "${CMAKE_SOURCE_DIR}/src/api/api.cpp" "${CMAKE_SOURCE_DIR}/src/api/api_database.cpp" + "${CMAKE_SOURCE_DIR}/src/api/bsa.cpp" "${CMAKE_SOURCE_DIR}/src/api/error_categories.cpp" "${CMAKE_SOURCE_DIR}/src/api/metadata/condition_evaluator.cpp" "${CMAKE_SOURCE_DIR}/src/api/metadata/conditional_metadata.cpp" @@ -227,6 +228,8 @@ set(LIBLOOT_INCLUDE_H_FILES set(LIBLOOT_SRC_API_H_FILES "${CMAKE_SOURCE_DIR}/src/api/api_database.h" + "${CMAKE_SOURCE_DIR}/src/api/bsa.h" + "${CMAKE_SOURCE_DIR}/src/api/bsa_detail.h" "${CMAKE_SOURCE_DIR}/src/api/metadata/condition_evaluator.h" "${CMAKE_SOURCE_DIR}/src/api/metadata/yaml/file.h" "${CMAKE_SOURCE_DIR}/src/api/metadata/yaml/group.h" diff --git a/docs/api/sorting.rst b/docs/api/sorting.rst index 7be5710e..02929cd2 100644 --- a/docs/api/sorting.rst +++ b/docs/api/sorting.rst @@ -59,19 +59,25 @@ can be corrected. Plugin overlap edges are then added. Two plugins overlap if they contain the same record, i.e. if they both edit the same record or if one edits a record the -other plugin adds. +other plugin adds. Plugins also overlap if they both load one or more BSAs and +the BSAs loaded by one plugin contain data for a file path that is also included +in the BSAs loaded by the other plugin. For each plugin, skip it if it overrides no records, otherwise iterate over all other plugins. -* If the plugin and other plugin override the same number of records, or do not - overlap, skip the other plugin. +* If the plugin and other plugin override the same number of records and the + same number of assets, or do not overlap, skip the other plugin. * Otherwise, add an edge from the plugin which overrides more records to the - plugin that overrides fewer records, unless that edge would cause a cycle. + plugin that overrides fewer records, unless that edge would cause a cycle. If + the plugins don't have overlapping records or override the same number of + records, the edge is added from the plugin that loads more assets via its + BSAs to the plugin that loads fewer assets. For Morrowind, identifying which records override others requires all of a plugin's masters to be installed, so if a plugin has missing masters, its total -record count is used in place of its override record count. +record count is used in place of its override record count. Morrowind plugins +also can't load BSAs, so they can't have overlapping assets. Finally, tie-break edges are added to ensure that sorting is consistent. For each plugin, iterate over all other plugins and add an edge between each pair of diff --git a/src/api/bsa.cpp b/src/api/bsa.cpp new file mode 100644 index 00000000..1fe3005b --- /dev/null +++ b/src/api/bsa.cpp @@ -0,0 +1,236 @@ +/* LOOT + + A load order optimisation tool for Oblivion, Skyrim, Fallout 3 and + Fallout: New Vegas. + + Copyright (C) 2022 Oliver Hamlet + + This file is part of LOOT. + + LOOT is free software: you can redistribute + it and/or modify it under the terms of the GNU General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + LOOT is distributed in the hope that it will + be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with LOOT. If not, see + . + */ + +#include "api/bsa.h" + +#include + +#include "api/bsa_detail.h" +#include "api/helpers/logging.h" + +namespace loot { +/* +BSA format documentation: + +- Oblivion: https://en.uesp.net/wiki/Oblivion_Mod:BSA_File_Format +- Fallout 3, Fallout New Vegas, Skyrim, Skyrim Special Edition: + https://en.uesp.net/wiki/Skyrim_Mod:Archive_File_Format + +*/ +constexpr std::array BSA_TYPE_ID = {'B', 'S', 'A', '\0'}; + +namespace bsa::v103 { +struct FolderRecord { + uint64_t nameHash{0}; + uint32_t fileCount{0}; + uint32_t fileRecordsOffset{0}; +}; + +std::map> GetAssetsInBethesdaArchive( + std::istream& in, + const Header& header) { + return detail::GetAssetsInBethesdaArchive(in, header); +} +} + +namespace bsa::v104 { +using bsa::v103::FolderRecord; + +using v103::GetAssetsInBethesdaArchive; +} + +namespace bsa::v105 { +struct FolderRecord { + uint64_t nameHash{0}; + uint32_t fileCount{0}; + uint32_t padding1{0}; + uint32_t fileRecordsOffset{0}; + uint32_t padding2{0}; +}; + +std::map> GetAssetsInBethesdaArchive( + std::istream& in, + const Header& header) { + return detail::GetAssetsInBethesdaArchive(in, header); +} +} + +bool DoFileNameHashSetsIntersect(const std::set& left, + const std::set& right) { + auto leftIt = left.begin(); + auto rightIt = right.begin(); + + while (leftIt != left.end() && rightIt != right.end()) { + if (*leftIt < *rightIt) { + ++leftIt; + } else if (*leftIt > *rightIt) { + ++rightIt; + } else { + return true; + } + } + + return false; +} + +std::map> GetAssetsInBethesdaArchive( + const std::filesystem::path& archivePath) { + // If parsing the BSA fails, log the error but don't throw an exception as + // an issue with one archive (which may just be invalid) shouldn't cause + // others not to be loaded. + + const auto logger = getLogger(); + + if (!std::filesystem::exists(archivePath)) { + if (logger) { + logger->error("Bethesda archive path \"{}\" does not exist!", + archivePath.u8string()); + throw std::runtime_error("Bethesda archive does not exist"); + } + } + + std::ifstream in(archivePath, std::ios::binary); + in.exceptions(std::ios::failbit | std::ios::badbit | + std::ios::eofbit); // Causes ifstream::failure to be thrown if + // a problem is encountered. + + bsa::Header header; + in.read(reinterpret_cast(&header), sizeof(bsa::Header)); + + // Validate the header. + if (header.typeId != BSA_TYPE_ID || + !(header.version == 103 || header.version == 104 || + header.version == 105) || + header.recordsOffset != 36) { + if (logger) { + logger->error("Bethesda archive at \"{}\" has an invalid header.", + archivePath.u8string()); + throw std::runtime_error("Bethesda archive has an invalid header"); + } + } + + if ((header.archiveFlags & 0x40) != 0) { + if (logger) { + logger->error("BSA file at \"{}\" uses big-endian numbers."); + } + throw std::runtime_error("BSA file uses big-endian numbers"); + } + + if (header.version == 103) { + return bsa::v103::GetAssetsInBethesdaArchive(in, header); + } + + if (header.version == 104) { + return bsa::v104::GetAssetsInBethesdaArchive(in, header); + } + + if (header.version == 105) { + return bsa::v105::GetAssetsInBethesdaArchive(in, header); + } + + if (logger) { + logger->error("Unrecognised BSA version {} in archive at \"{}\"", + header.version, + archivePath.u8string()); + } + + throw std::runtime_error("BSA file has an unrecognised version"); +} + +std::map> GetAssetsInBethesdaArchives( + const std::vector& archivePaths) { + const auto logger = getLogger(); + + std::map> archiveAssets; + + for (const auto& archivePath : archivePaths) { + try { + if (logger) { + logger->trace( + "Getting assets loaded from the Bethesda archive at \"{}\"", + archivePath.u8string()); + } + + const auto assets = GetAssetsInBethesdaArchive(archivePath); + for (const auto& asset : assets) { + const auto folderResult = archiveAssets.insert(asset); + if (!folderResult.second) { + // Folder already exists, add the files to its set. + // Don't just insert the range, as it would be good to + // log if a file's hash is already present - you wouldn't + // expect the same file to appear in the same folder in + // two different BSAs loaded by the same plugin. + /*result.first->second.insert(asset.second.begin(), + asset.second.end());*/ + for (const auto& fileNameHash : asset.second) { + const auto fileResult = + folderResult.first->second.insert(fileNameHash); + if (!fileResult.second && logger) { + logger->warn( + "The folder and file with hashes {:x} and {:x} in \"{}\" are " + "present in another BSA.", + asset.first, + fileNameHash, + archivePath.u8string()); + } + } + } + } + } catch (const std::exception& e) { + if (logger) { + logger->error( + "Caught exception while trying to read Bethesda archive file " + "at \"{}\": {}", + archivePath.u8string(), + e.what()); + } + } + } + + return archiveAssets; +} + +bool DoAssetsIntersect(const std::map>& left, + const std::map>& right) { + auto leftIt = left.begin(); + auto rightIt = right.begin(); + + while (leftIt != left.end() && rightIt != right.end()) { + if (leftIt->first < rightIt->first) { + ++leftIt; + } else if (leftIt->first > rightIt->first) { + ++rightIt; + } else if (DoFileNameHashSetsIntersect(leftIt->second, rightIt->second)) { + return true; + } else { + // The folder hashes are equal but they don't contain any of the same + // file hashes, move on to the next folder. It doesn't matter which + // iterator gets incremented. + ++leftIt; + } + } + + return false; +} +} diff --git a/src/api/bsa.h b/src/api/bsa.h new file mode 100644 index 00000000..73d5df46 --- /dev/null +++ b/src/api/bsa.h @@ -0,0 +1,44 @@ +/* LOOT + + A load order optimisation tool for Oblivion, Skyrim, Fallout 3 and + Fallout: New Vegas. + + Copyright (C) 2022 Oliver Hamlet + + This file is part of LOOT. + + LOOT is free software: you can redistribute + it and/or modify it under the terms of the GNU General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + LOOT is distributed in the hope that it will + be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with LOOT. If not, see + . + */ +#ifndef LOOT_API_BSA +#define LOOT_API_BSA + +#include +#include +#include +#include +#include + +namespace loot { +std::map> GetAssetsInBethesdaArchive( + const std::filesystem::path& archivePath); + +std::map> GetAssetsInBethesdaArchives( + const std::vector& archivePaths); + +bool DoAssetsIntersect(const std::map>& left, + const std::map>& right); +} + +#endif diff --git a/src/api/bsa_detail.h b/src/api/bsa_detail.h new file mode 100644 index 00000000..a851a17f --- /dev/null +++ b/src/api/bsa_detail.h @@ -0,0 +1,144 @@ +/* LOOT + + A load order optimisation tool for Oblivion, Skyrim, Fallout 3 and + Fallout: New Vegas. + + Copyright (C) 2022 Oliver Hamlet + + This file is part of LOOT. + + LOOT is free software: you can redistribute + it and/or modify it under the terms of the GNU General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + LOOT is distributed in the hope that it will + be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with LOOT. If not, see + . + */ +#ifndef LOOT_API_BSA_DETAIL +#define LOOT_API_BSA_DETAIL + +#include +#include +#include +#include +#include + +#include "api/helpers/logging.h" + +namespace loot::bsa { +struct Header { + std::array typeId; // Should always be "BSA\0" + uint32_t version{0}; // 103 (0x67) for TES4, 104 (0x68) for FO3, FONV, TES5, + // 105 + // (0x69) for TES5SE. + uint32_t recordsOffset{0}; + uint32_t archiveFlags{0}; + uint32_t folderCount{0}; + uint32_t totalFileCount{0}; + uint32_t totalFolderNamesLength{0}; + uint32_t totalFileNamesLength{0}; + uint32_t contentTypeFlags{0}; +}; + +struct FileRecord { + uint64_t nameHash{0}; + uint32_t dataLength{0}; + uint32_t dataOffset{0}; +}; +} + +namespace loot::bsa::detail { +template +std::map> GetAssetsInBethesdaArchive( + std::istream& in, + const Header& header) { + const auto logger = getLogger(); + + std::vector folderRecords(header.folderCount); + in.read(reinterpret_cast(folderRecords.data()), + sizeof(FolderRecord) * folderRecords.size()); + + // The next block consists of per-folder subblocks that are each a + // byte containing the folder name length, the null-terminated folder name + // and then the file records for that folder. + const auto fileRecordsSize = header.folderCount + + header.totalFolderNamesLength + + sizeof(FileRecord) * header.totalFileCount; + std::vector fileRecordsBytes(fileRecordsSize); + in.read(reinterpret_cast(fileRecordsBytes.data()), fileRecordsSize); + + // For each folder record, store its hash with the hashes of the files in that + // folder. + std::map> folderFileHashes; + + // FolderRecord.fileRecordsOffset is relative to this baseline. In the file + // fileRecordsOffset - header.totalFileNamesLength is the start off the + // folder's subblock relative to the start of the file, but the baseline is + // from the start of the fileRecords vector. + const auto folderRecordOffsetBaseline = + sizeof(Header) + sizeof(FolderRecord) * header.folderCount + + header.totalFileNamesLength; + + for (const auto& folderRecord : folderRecords) { + const auto folderHash = folderRecord.nameHash; + + const auto folderResult = + folderFileHashes.emplace(folderHash, std::set()); + + if (!folderResult.second) { + if (logger) { + logger->warn("Folder name hash {} is already in map", folderHash); + } + throw std::runtime_error("Unexpected folder name hash collision"); + } + + size_t fileRecordsOffset = 0; + if ((header.archiveFlags & 0x1) == 0) { + // Directory names are not included. + fileRecordsOffset = + folderRecord.fileRecordsOffset - folderRecordOffsetBaseline; + } else { + // Directory names are included. + const auto folderNameLengthOffset = + folderRecord.fileRecordsOffset - folderRecordOffsetBaseline; + + const auto folderNameLength = fileRecordsBytes.at(folderNameLengthOffset); + + // The real file records offset. + fileRecordsOffset = folderNameLengthOffset + 1 + folderNameLength; + } + + for (size_t i = 0; i < folderRecord.fileCount; ++i) { + const auto fileRecordOffset = + fileRecordsBytes.data() + fileRecordsOffset + i * sizeof(FileRecord); + + const FileRecord* fileRecord = + reinterpret_cast(fileRecordOffset); + + const auto result = + folderResult.first->second.insert(fileRecord->nameHash); + + if (!result.second) { + if (logger) { + logger->warn( + "File name hash {} is already in the set for folder name hash {}", + fileRecord->nameHash, + folderHash); + } + throw std::runtime_error("Unexpected file name hash collision"); + } + } + } + + return folderFileHashes; +} +} + +#endif diff --git a/src/api/plugin.cpp b/src/api/plugin.cpp index 8748c7c9..2d39a7e0 100644 --- a/src/api/plugin.cpp +++ b/src/api/plugin.cpp @@ -26,7 +26,9 @@ #include #include +#include +#include "api/bsa.h" #include "api/game/game.h" #include "api/helpers/crc.h" #include "api/helpers/logging.h" @@ -34,6 +36,107 @@ #include "loot/exception/file_access_error.h" namespace loot { +std::filesystem::path ReplaceExtension(std::filesystem::path path, + const std::string& newExtension) { + return path.replace_extension(std::filesystem::u8path(newExtension)); +} + +std::filesystem::path GetTexturesArchivePath(std::filesystem::path pluginPath, + const std::string& newExtension) { + // replace_extension() with no argument just removes the existing extension. + pluginPath.replace_extension(); + pluginPath += " - Textures" + newExtension; + return pluginPath; +} + +bool equivalent(const std::filesystem::path& path1, + const std::filesystem::path& path2) { + // If the paths are identical, they've got to be equivalent, + // it doesn't matter if the paths exist or not. + if (path1 == path2) { + return true; + } + // If the paths are not identical, the filesystem might be case-insensitive + // so check with the filesystem. + try { + return std::filesystem::equivalent(path1, path2); + } catch (const std::filesystem::filesystem_error&) { + // One of the paths checked for equivalence doesn't exist, + // so they can't be equivalent. + return false; + } catch (const std::system_error&) { + // This can be thrown if one or both of the paths contains a character + // that can't be represented in Windows' multi-byte code page (e.g. + // Windows-1252), even though Unicode paths shouldn't be a problem, + // and throwing system_error is undocumented. Seems like a bug in MSVC's + // implementation. + return false; + } +} + +std::vector FindAssociatedArchives( + const GameType gameType, + const GameCache& gameCache, + const std::filesystem::path& pluginPath) { + std::vector paths; + + if (gameType == GameType::tes3) { + return paths; + } + + const auto archiveExtension = GetArchiveFileExtension(gameType); + + if (gameType == GameType::tes5) { + // Skyrim (non-SE) plugins can only load BSAs that have exactly the same + // basename, ignoring file extensions. + const auto archiveFilename = ReplaceExtension(pluginPath, archiveExtension); + + if (std::filesystem::exists(archiveFilename)) { + paths.push_back(archiveFilename); + } + } else if (gameType == GameType::tes5se || gameType == GameType::tes5vr) { + // Skyrim SE can load BSAs that have exactly the same + // basename, ignoring file extensions, and also BSAs with filenames of + // the form " - Textures.bsa" (case-insensitively). + // This assumes that Skyrim VR works the same way as Skyrim SE. + const auto archiveFilename = ReplaceExtension(pluginPath, archiveExtension); + const auto texturesArchiveFilename = + GetTexturesArchivePath(pluginPath, archiveExtension); + + if (std::filesystem::exists(archiveFilename)) { + paths.push_back(archiveFilename); + } + + if (std::filesystem::exists(texturesArchiveFilename)) { + paths.push_back(texturesArchiveFilename); + } + } else if (gameType != GameType::tes4 || + boost::iends_with(pluginPath.filename().u8string(), ".esp")) { + // Oblivion .esp files and FO3, FNV, FO4 plugins can load archives which + // begin with the plugin basename. + // This assumes that FO4 VR works the same way as FO4. + + const auto basenameLength = pluginPath.stem().native().length(); + const auto pluginExtension = pluginPath.extension().native(); + + for (const auto& archivePath : gameCache.GetArchivePaths()) { + // Need to check if it starts with the given plugin's basename, + // but case insensitively. This is hard to do accurately, so + // instead check if the plugin with the same length basename and + // and the given plugin's file extension is equivalent. + const auto bsaPluginFilename = + archivePath.filename().native().substr(0, basenameLength) + + pluginExtension; + const auto bsaPluginPath = pluginPath.parent_path() / bsaPluginFilename; + if (loot::equivalent(pluginPath, bsaPluginPath)) { + paths.push_back(archivePath); + } + } + } + + return paths; +} + Plugin::Plugin(const GameType gameType, const GameCache& gameCache, std::filesystem::path pluginPath, @@ -43,7 +146,6 @@ Plugin::Plugin(const GameType gameType, std::unique_ptr<::Plugin, decltype(&esp_plugin_free)>(nullptr, esp_plugin_free)), isEmpty_(true), - loadsArchive_(false), overrideRecordCount_(0) { auto logger = getLogger(); @@ -62,6 +164,8 @@ Plugin::Plugin(const GameType gameType, "\" is empty. esplugin error code: " + std::to_string(ret)); } + archivePaths_ = FindAssociatedArchives(gameType, gameCache, pluginPath); + if (!headerOnly) { crc_ = GetCrc32(pluginPath); @@ -72,10 +176,20 @@ Plugin::Plugin(const GameType gameType, "Error counting override records in \"" + name_ + "\". esplugin error code: " + std::to_string(ret)); } + + // Get the assets in the BSAs that this plugin loads. + auto assets = GetAssetsInBethesdaArchives(archivePaths_); + std::swap(archiveAssets_, assets); + + if (logger) { + logger->debug( + "Plugin file \"{}\" loads {} assets from Bethesda archives", + name_, + GetAssetCount()); + } } tags_ = ExtractBashTags(GetDescription()); - loadsArchive_ = LoadsArchive(gameType, gameCache, pluginPath); } catch (const std::exception& e) { if (logger) { logger->error( @@ -164,7 +278,7 @@ bool Plugin::IsValidAsLightPlugin() const { bool Plugin::IsEmpty() const { return isEmpty_; } -bool Plugin::LoadsArchive() const { return loadsArchive_; } +bool Plugin::LoadsArchive() const { return !archivePaths_.empty(); } bool Plugin::DoFormIDsOverlap(const PluginInterface& plugin) const { try { @@ -242,6 +356,36 @@ uint32_t Plugin::GetRecordAndGroupCount() const { return recordAndGroupCount; } +size_t Plugin::GetAssetCount() const { + return std::accumulate( + archiveAssets_.begin(), + archiveAssets_.end(), + size_t{0}, + [](const size_t& a, const auto& b) { return a + b.second.size(); }); +} + +bool Plugin::DoAssetsOverlap(const PluginSortingInterface& plugin) const { + if (archiveAssets_.empty()) { + return false; + } + + try { + const auto& otherPlugin = dynamic_cast(plugin); + + return DoAssetsIntersect(archiveAssets_, otherPlugin.archiveAssets_); + } catch (std::bad_cast&) { + auto logger = getLogger(); + if (logger) { + logger->error( + "Tried to check how many FormIDs overlapped with a non-Plugin " + "implementation of PluginSortingInterface."); + } + throw std::invalid_argument( + "Tried to check how many FormIDs overlapped with a non-Plugin " + "implementation of PluginSortingInterface."); + } +} + bool Plugin::IsValid(const GameType gameType, const std::filesystem::path& pluginPath) { // Check that the file has a valid extension. @@ -325,98 +469,6 @@ std::string GetArchiveFileExtension(const GameType gameType) { return ".bsa"; } -std::filesystem::path replaceExtension(std::filesystem::path path, - const std::string& newExtension) { - return path.replace_extension(std::filesystem::u8path(newExtension)); -} - -std::filesystem::path getTexturesArchivePath(std::filesystem::path pluginPath, - const std::string& newExtension) { - // replace_extension() with no argument just removes the existing extension. - pluginPath.replace_extension(); - pluginPath += " - Textures" + newExtension; - return pluginPath; -} - -bool equivalent(const std::filesystem::path& path1, - const std::filesystem::path& path2) { - // If the paths are identical, they've got to be equivalent, - // it doesn't matter if the paths exist or not. - if (path1 == path2) { - return true; - } - // If the paths are not identical, the filesystem might be case-insensitive - // so check with the filesystem. - try { - return std::filesystem::equivalent(path1, path2); - } catch (const std::filesystem::filesystem_error&) { - // One of the paths checked for equivalence doesn't exist, - // so they can't be equivalent. - return false; - } catch (const std::system_error&) { - // This can be thrown if one or both of the paths contains a character - // that can't be represented in Windows' multi-byte code page (e.g. - // Windows-1252), even though Unicode paths shouldn't be a problem, - // and throwing system_error is undocumented. Seems like a bug in MSVC's - // implementation. - return false; - } -} - -// Get whether the plugin loads an archive (BSA/BA2) or not. -bool Plugin::LoadsArchive(const GameType gameType, - const GameCache& gameCache, - const std::filesystem::path& pluginPath) { - if (gameType == GameType::tes3) { - return false; - } - - const auto archiveExtension = GetArchiveFileExtension(gameType); - - if (gameType == GameType::tes5) { - // Skyrim (non-SE) plugins can only load BSAs that have exactly the same - // basename, ignoring file extensions. - auto archiveFilename = replaceExtension(pluginPath, archiveExtension); - - return std::filesystem::exists(archiveFilename); - } else if (gameType == GameType::tes5se || gameType == GameType::tes5vr) { - // Skyrim SE can load BSAs that have exactly the same - // basename, ignoring file extensions, and also BSAs with filenames of - // the form " - Textures.bsa" (case-insensitively). - // I'm assuming Skyrim VR works the same way as Skyrim SE. - auto archiveFilename = replaceExtension(pluginPath, archiveExtension); - auto texturesArchiveFilename = - getTexturesArchivePath(pluginPath, archiveExtension); - - return std::filesystem::exists(archiveFilename) || - std::filesystem::exists(texturesArchiveFilename); - } else if (gameType != GameType::tes4 || - boost::iends_with(pluginPath.filename().u8string(), ".esp")) { - // Oblivion .esp files and FO3, FNV, FO4 plugins can load archives which - // begin with the plugin basename. - // I'm assuming that FO4 VR works the same way as FO4. - - auto basenameLength = pluginPath.stem().native().length(); - auto pluginExtension = pluginPath.extension().native(); - - for (const auto& archivePath : gameCache.GetArchivePaths()) { - // Need to check if it starts with the given plugin's basename, - // but case insensitively. This is hard to do accurately, so - // instead check if the plugin with the same length basename and - // and the given plugin's file extension is equivalent. - auto bsaPluginFilename = - archivePath.filename().native().substr(0, basenameLength) + - pluginExtension; - auto bsaPluginPath = pluginPath.parent_path() / bsaPluginFilename; - if (loot::equivalent(pluginPath, bsaPluginPath)) { - return true; - } - } - } - - return false; -} - unsigned int Plugin::GetEspluginGameId(GameType gameType) { switch (gameType) { case GameType::tes3: diff --git a/src/api/plugin.h b/src/api/plugin.h index ff1c9759..3ee8e36e 100644 --- a/src/api/plugin.h +++ b/src/api/plugin.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,9 @@ public: virtual size_t GetOverlapSize( const std::vector& plugins) const = 0; + + virtual size_t GetAssetCount() const = 0; + virtual bool DoAssetsOverlap(const PluginSortingInterface& plugin) const = 0; }; class Plugin final : public PluginSortingInterface { @@ -78,6 +82,9 @@ public: size_t GetOverrideRecordCount() const override; uint32_t GetRecordAndGroupCount() const override; + size_t GetAssetCount() const; + bool DoAssetsOverlap(const PluginSortingInterface& plugin) const; + // Validity checks. static bool IsValid(const GameType gameType, const std::filesystem::path& pluginPath); @@ -89,20 +96,18 @@ private: bool headerOnly); std::string GetDescription() const; - static bool LoadsArchive(const GameType gameType, - const GameCache& gameCache, - const std::filesystem::path& pluginPath); static unsigned int GetEspluginGameId(GameType gameType); std::string name_; std::unique_ptr<::Plugin, decltype(&esp_plugin_free)> esPlugin; bool isEmpty_; // Does the plugin contain any records other than the TES4 // header? - bool loadsArchive_; size_t overrideRecordCount_; std::optional version_; // Obtained from description field. std::optional crc_; std::vector tags_; + std::vector archivePaths_; + std::map> archiveAssets_; }; std::string GetArchiveFileExtension(const GameType gameType); diff --git a/src/api/sorting/plugin_graph.cpp b/src/api/sorting/plugin_graph.cpp index 422eb3f5..ddc743f4 100644 --- a/src/api/sorting/plugin_graph.cpp +++ b/src/api/sorting/plugin_graph.cpp @@ -777,12 +777,14 @@ void PluginGraph::AddOverlapEdges() { for (auto [vit, vitend] = GetVertices(); vit != vitend; ++vit) { const auto vertex = *vit; const auto& plugin = GetPlugin(vertex); + const auto pluginRecordCount = plugin.GetOverrideRecordCount(); + const auto pluginAssetCount = plugin.GetAssetCount(); - if (plugin.GetOverrideRecordCount() == 0) { + if (pluginRecordCount == 0 && pluginAssetCount == 0) { if (logger) { logger->debug( "Skipping vertex for \"{}\": the plugin contains no override " - "records.", + "records and loads no assets.", plugin.GetName()); } continue; @@ -792,21 +794,56 @@ void PluginGraph::AddOverlapEdges() { const auto otherVertex = *vit2; const auto& otherPlugin = GetPlugin(otherVertex); - if (vertex == otherVertex || EdgeExists(vertex, otherVertex) || - EdgeExists(otherVertex, vertex) || - plugin.GetOverrideRecordCount() == otherPlugin.GetOverrideRecordCount() || - !plugin.DoRecordsOverlap(otherPlugin)) { + // Don't add an edge between these two plugins if one already + // exists (only check direct edges and not paths for efficiency). + if (EdgeExists(vertex, otherVertex) || EdgeExists(otherVertex, vertex)) { continue; } - const auto thisPluginOverridesMoreRecords = - plugin.GetOverrideRecordCount() > otherPlugin.GetOverrideRecordCount(); - const auto fromVertex = - thisPluginOverridesMoreRecords ? vertex : otherVertex; - const auto toVertex = thisPluginOverridesMoreRecords ? otherVertex : vertex; + // Two plugins can overlap due to overriding the same records, + // or by loading assets from BSAs/BA2s that have the same path. + // If records overlap, the plugin that overrides more records + // should load earlier. + // If assets overlap, the plugin that loads more assets should + // load earlier. + // If two plugins have overlapping records and assets and one + // overrides more records but loads fewer assets than the other, + // the fact it overrides more records should take precedence + // (records are more significant than assets). + // I.e. if two plugins don't have overlapping records, check their + // assets, otherwise only check their assets if their override + // record counts are equal. - if (!PathExists(toVertex, fromVertex)) + auto thisPluginLoadsFirst = false; + + const auto otherPluginRecordCount = otherPlugin.GetOverrideRecordCount(); + + if (pluginRecordCount == otherPluginRecordCount || + !plugin.DoRecordsOverlap(otherPlugin)) { + // Records don't overlap, or override the same number of records, + // check assets. + // No records overlap, check assets. + const auto otherPluginAssetCount = otherPlugin.GetAssetCount(); + if (pluginAssetCount == otherPluginAssetCount || + !plugin.DoAssetsOverlap(otherPlugin)) { + // Assets don't overlap or both plugins load the same number of + // assets, don't add an edge. + continue; + } else { + thisPluginLoadsFirst = pluginAssetCount > otherPluginAssetCount; + } + } else { + // Records overlap and override different numbers of records. + // Load this plugin first if it overrides more records. + thisPluginLoadsFirst = pluginRecordCount > otherPluginRecordCount; + } + + const auto fromVertex = thisPluginLoadsFirst ? vertex : otherVertex; + const auto toVertex = thisPluginLoadsFirst ? otherVertex : vertex; + + if (!PathExists(toVertex, fromVertex)) { AddEdge(fromVertex, toVertex, EdgeType::overlap); + } } } } diff --git a/src/api/sorting/plugin_sorting_data.cpp b/src/api/sorting/plugin_sorting_data.cpp index e2fa005b..1f0834d5 100644 --- a/src/api/sorting/plugin_sorting_data.cpp +++ b/src/api/sorting/plugin_sorting_data.cpp @@ -137,6 +137,15 @@ bool PluginSortingData::DoRecordsOverlap( plugin_->DoFormIDsOverlap(*plugin.plugin_); } +size_t PluginSortingData::GetAssetCount() const { + return plugin_ == nullptr ? 0 : plugin_->GetAssetCount(); +} + +bool PluginSortingData::DoAssetsOverlap(const PluginSortingData& plugin) const { + return plugin_ != nullptr && plugin.plugin_ != nullptr && + plugin_->DoAssetsOverlap(*plugin.plugin_); +} + std::string PluginSortingData::GetGroup() const { return group_; } std::unordered_set PluginSortingData::GetAfterGroupPlugins() diff --git a/src/api/sorting/plugin_sorting_data.h b/src/api/sorting/plugin_sorting_data.h index 06111cec..969ef9c1 100644 --- a/src/api/sorting/plugin_sorting_data.h +++ b/src/api/sorting/plugin_sorting_data.h @@ -55,6 +55,9 @@ public: size_t GetOverrideRecordCount() const; bool DoRecordsOverlap(const PluginSortingData& plugin) const; + size_t GetAssetCount() const; + bool DoAssetsOverlap(const PluginSortingData& plugin) const; + std::string GetGroup() const; std::unordered_set GetAfterGroupPlugins() const; diff --git a/src/tests/api/internals/bsa_test.h b/src/tests/api/internals/bsa_test.h new file mode 100644 index 00000000..1b3a7bf3 --- /dev/null +++ b/src/tests/api/internals/bsa_test.h @@ -0,0 +1,157 @@ +/* LOOT + +A load order optimisation tool for Oblivion, Skyrim, Fallout 3 and +Fallout: New Vegas. + +Copyright (C) 2014-2016 WrinklyNinja + +This file is part of LOOT. + +LOOT is free software: you can redistribute +it and/or modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. + +LOOT is distributed in the hope that it will +be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with LOOT. If not, see +. +*/ + +#ifndef LOOT_TESTS_API_INTERNALS_BSA_TEST +#define LOOT_TESTS_API_INTERNALS_BSA_TEST + +#include + +#include "api/bsa.h" + +namespace loot::test { +TEST(GetAssetsInBethesdaArchive, shouldSupportV103BSAs) { + const auto path = std::filesystem::u8path("./Oblivion/Data/Blank.bsa"); + + const auto assets = GetAssetsInBethesdaArchive(path); + + size_t filesCount = 0; + for (const auto& folder : assets) { + filesCount += folder.second.size(); + } + + EXPECT_EQ(1, assets.size()); + EXPECT_EQ(1, filesCount); + EXPECT_EQ(0, assets.begin()->first); + EXPECT_EQ(1, assets.at(0).size()); + EXPECT_EQ(0x4670B6836C077365, *assets.at(0).begin()); +} + +TEST(GetAssetsInBethesdaArchive, shouldSupportV104BSAs) { + const auto path = std::filesystem::u8path("./Skyrim/Data/Blank.bsa"); + + const auto assets = GetAssetsInBethesdaArchive(path); + + size_t filesCount = 0; + for (const auto& folder : assets) { + filesCount += folder.second.size(); + } + + EXPECT_EQ(1, assets.size()); + EXPECT_EQ(1, filesCount); + EXPECT_EQ(0x2E01002E, assets.begin()->first); + EXPECT_EQ(1, assets.at(0x2E01002E).size()); + EXPECT_EQ(0x4670B6836C077365, *assets.at(0x2E01002E).begin()); +} + +TEST(GetAssetsInBethesdaArchive, shouldSupportV105BSAs) { + const auto path = std::filesystem::u8path("./SkyrimSE/Data/Blank.bsa"); + + const auto assets = GetAssetsInBethesdaArchive(path); + + size_t filesCount = 0; + for (const auto& folder : assets) { + filesCount += folder.second.size(); + } + + EXPECT_EQ(1, assets.size()); + EXPECT_EQ(1, filesCount); + EXPECT_EQ(0xB68102C964176E73, assets.begin()->first); + EXPECT_EQ(1, assets.at(0xB68102C964176E73).size()); + EXPECT_EQ(0x4670B6836C077365, *assets.at(0xB68102C964176E73).begin()); +} + +TEST(GetAssetsInBethesdaArchive, shouldThrowIfFileCannotBeOpened) { + const auto path = std::filesystem::u8path("invalid.bsa"); + + EXPECT_THROW(GetAssetsInBethesdaArchive(path), std::runtime_error); +} + +TEST(GetAssetsInBethesdaArchives, shouldSkipFilesThatCannotBeRead) { + std::vector paths( + {std::filesystem::u8path("invalid.bsa"), + std::filesystem::u8path("./Skyrim/Data/Blank.bsa")}); + + const auto assets = GetAssetsInBethesdaArchives(paths); + + size_t filesCount = 0; + for (const auto& folder : assets) { + filesCount += folder.second.size(); + } + + EXPECT_EQ(1, assets.size()); + EXPECT_EQ(1, filesCount); + EXPECT_EQ(0x2E01002E, assets.begin()->first); + EXPECT_EQ(1, assets.begin()->second.size()); + EXPECT_EQ(0x4670B6836C077365, *assets.begin()->second.begin()); +} + +TEST(GetAssetsInBethesdaArchives, shouldCombineAssetsFromEachLoadedArchive) { + std::vector paths( + {std::filesystem::u8path("./Oblivion/Data/Blank.bsa"), + std::filesystem::u8path("./Skyrim/Data/Blank.bsa"), + std::filesystem::u8path("./SkyrimSE/Data/Blank.bsa")}); + + const auto assets = GetAssetsInBethesdaArchives(paths); + + size_t filesCount = 0; + for (const auto& folder : assets) { + filesCount += folder.second.size(); + } + + EXPECT_EQ(3, assets.size()); + EXPECT_EQ(3, filesCount); + + EXPECT_EQ(1, assets.at(0).size()); + EXPECT_EQ(0x4670B6836C077365, *assets.at(0).begin()); + + EXPECT_EQ(1, assets.at(0x2E01002E).size()); + EXPECT_EQ(0x4670B6836C077365, *assets.at(0x2E01002E).begin()); + + EXPECT_EQ(1, assets.at(0xB68102C964176E73).size()); + EXPECT_EQ(0x4670B6836C077365, *assets.at(0xB68102C964176E73).begin()); +} + +TEST(DoAssetsIntersect, shouldReturnTrueIfTheSameFileExistsInTheSameFolder) { + const auto path = std::filesystem::u8path("./Oblivion/Data/Blank.bsa"); + + const auto assets = GetAssetsInBethesdaArchive(path); + + EXPECT_TRUE(DoAssetsIntersect(assets, assets)); +} + +TEST(DoAssetsIntersect, + shouldReturnFalseIfTheSameFileExistsInDifferentFolders) { + const auto path1 = std::filesystem::u8path("./Oblivion/Data/Blank.bsa"); + const auto assets1 = GetAssetsInBethesdaArchive(path1); + + const auto path2 = std::filesystem::u8path("./Skyrim/Data/Blank.bsa"); + const auto assets2 = GetAssetsInBethesdaArchive(path2); + + EXPECT_EQ(*assets2.at(0x2E01002E).begin(), *assets1.at(0).begin()); + + EXPECT_FALSE(DoAssetsIntersect(assets1, assets2)); +} +} + +#endif diff --git a/src/tests/api/internals/main.cpp b/src/tests/api/internals/main.cpp index ff9ea1bf..98171cf9 100644 --- a/src/tests/api/internals/main.cpp +++ b/src/tests/api/internals/main.cpp @@ -22,6 +22,7 @@ . */ +#include "tests/api/internals/bsa_test.h" #include "tests/api/internals/game/game_cache_test.h" #include "tests/api/internals/game/game_test.h" #include "tests/api/internals/game/load_order_handler_test.h" diff --git a/src/tests/api/internals/plugin_test.h b/src/tests/api/internals/plugin_test.h index 9f759f59..b1e7e227 100644 --- a/src/tests/api/internals/plugin_test.h +++ b/src/tests/api/internals/plugin_test.h @@ -70,9 +70,23 @@ protected: std::filesystem::copy(dataPath / blankEsp, dataPath / blankEsl)); } + // Copy across archive files. + const auto blankMasterDependentArchive = + "Blank - Master Dependent" + GetArchiveFileExtension(GetParam()); + if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + out.open(dataPath / blankArchive); + out.close(); + } else { + copyPlugin(getSourcePluginsPath(), blankArchive); + + // Also create a copy for Blank - Master Dependent.esp to test overlap. + std::filesystem::copy_file(getSourcePluginsPath() / blankArchive, + dataPath / blankMasterDependentArchive); + ASSERT_TRUE( + std::filesystem::exists(dataPath / blankMasterDependentArchive)); + } + // Create dummy archive files. - out.open(dataPath / blankArchive); - out.close(); out.open(dataPath / blankSuffixArchive); out.close(); @@ -91,6 +105,7 @@ protected: out.close(); game_.GetCache().CacheArchivePaths({dataPath / blankArchive, + dataPath / blankMasterDependentArchive, dataPath / blankSuffixArchive, dataPath / nonAsciiArchivePath, dataPath / nonAsciiPrefixArchivePath}); @@ -163,6 +178,11 @@ public: const std::vector&) const override { return 0; }; + + size_t GetAssetCount() const override { return 0; }; + bool DoAssetsOverlap(const PluginSortingInterface&) const override { + return true; + } }; // Pass an empty first argument, as it's a prefix for the test instantation, @@ -395,7 +415,7 @@ TEST_P(PluginTest, loadsArchiveShouldReturnFalseForAPluginThatDoesNotLoadAnArchive) { EXPECT_FALSE(Plugin(game_.Type(), game_.GetCache(), - game_.DataPath() / blankMasterDependentEsp, + game_.DataPath() / blankDifferentMasterDependentEsp, true) .LoadsArchive()); } @@ -562,6 +582,87 @@ TEST_P(PluginTest, getRecordAndGroupCountShouldReturnTheHeaderFieldValue) { } } +TEST_P(PluginTest, + getAssetCountShouldReturnNumberOfFilesInArchivesLoadedByPlugin) { + const auto assetCount = + Plugin(game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, false) + .GetAssetCount(); + + if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + EXPECT_EQ(0, assetCount); + } else { + EXPECT_EQ(1, assetCount); + } +} + +TEST_P(PluginTest, getAssetCountShouldReturnZeroIfOnlyPluginHeaderWasLoaded) { + const auto assetCount = + Plugin(game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, true) + .GetAssetCount(); + + EXPECT_EQ(0, assetCount); +} + +TEST_P(PluginTest, + doAssetsOverlapShouldReturnFalseOrThrowIfTheArgumentIsNotAPluginObject) { + Plugin plugin1( + game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, false); + OtherPluginType plugin2; + + if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + EXPECT_FALSE(plugin1.DoAssetsOverlap(plugin2)); + } else { + EXPECT_THROW(plugin1.DoAssetsOverlap(plugin2), std::invalid_argument); + } + EXPECT_TRUE(plugin2.DoAssetsOverlap(plugin1)); +} + +TEST_P(PluginTest, + doAssetsOverlapShouldReturnFalseForTwoPluginsWithOnlyHeadersLoaded) { + Plugin plugin1( + game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, true); + Plugin plugin2(game_.Type(), + game_.GetCache(), + game_.DataPath() / blankMasterDependentEsp, + true); + + EXPECT_FALSE(plugin1.DoAssetsOverlap(plugin2)); + EXPECT_FALSE(plugin2.DoAssetsOverlap(plugin1)); +} + +TEST_P(PluginTest, + doAssetsOverlapShouldReturnFalseIfThePluginsDoNotLoadTheSameAssetPath) { + Plugin plugin1( + game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, false); + // Blank - Different.esp does not load any assets. + Plugin plugin2(game_.Type(), + game_.GetCache(), + game_.DataPath() / blankDifferentEsp, + false); + + EXPECT_FALSE(plugin1.DoAssetsOverlap(plugin2)); + EXPECT_FALSE(plugin2.DoAssetsOverlap(plugin1)); +} + +TEST_P(PluginTest, + doAssetsOverlapShouldReturnTrueIfThePluginsLoadTheSameAssetPath) { + Plugin plugin1( + game_.Type(), game_.GetCache(), game_.DataPath() / blankEsp, false); + Plugin plugin2(game_.Type(), + game_.GetCache(), + game_.DataPath() / blankMasterDependentEsp, + false); + + if (GetParam() == GameType::tes3 || GetParam() == GameType::fo4) { + // Morrowind plugins can't load assets. + EXPECT_FALSE(plugin1.DoAssetsOverlap(plugin2)); + EXPECT_FALSE(plugin2.DoAssetsOverlap(plugin1)); + } else { + EXPECT_TRUE(plugin1.DoAssetsOverlap(plugin2)); + EXPECT_TRUE(plugin2.DoAssetsOverlap(plugin1)); + } +} + TEST_P(PluginTest, hasPluginFileExtensionShouldBeTrueIfFileEndsInDotEspOrDotEsm) { EXPECT_TRUE(hasPluginFileExtension("file.esp", GetParam())); diff --git a/src/tests/api/internals/sorting/plugin_graph_test.h b/src/tests/api/internals/sorting/plugin_graph_test.h index 08dc7846..7f08f235 100644 --- a/src/tests/api/internals/sorting/plugin_graph_test.h +++ b/src/tests/api/internals/sorting/plugin_graph_test.h @@ -31,12 +31,356 @@ along with LOOT. If not, see namespace loot { namespace test { -TEST(PluginGraph, topologicalSortWithNoLoadedPluginsShouldReturnAnEmptyList) { +namespace plugingraph { +class TestPlugin : public PluginSortingInterface { +public: + TestPlugin(const std::string& name) : name_(name) {} + + std::string GetName() const override { return name_; } + + std::optional GetHeaderVersion() const override { + return std::optional(); + } + + std::optional GetVersion() const override { + return std::optional(); + } + + std::vector GetMasters() const override { + return std::vector(); + } + + std::vector GetBashTags() const override { return std::vector(); } + + std::optional GetCRC() const override { + return std::optional(); + } + + bool IsMaster() const override { return false; } + + bool IsLightPlugin() const override { return false; } + + bool IsValidAsLightPlugin() const override { return false; } + + bool IsEmpty() const override { return false; } + + bool LoadsArchive() const override { return false; } + + bool DoFormIDsOverlap(const PluginInterface& plugin) const override { + const auto otherPlugin = dynamic_cast(&plugin); + return recordsOverlapWith.count(&plugin) != 0 || + otherPlugin->recordsOverlapWith.count(this) != 0; + } + + size_t GetOverrideRecordCount() const override { + return overrideRecordCount_; + } + + uint32_t GetRecordAndGroupCount() const override { return uint32_t(); } + + size_t GetOverlapSize( + const std::vector&) const override { + return size_t(); + } + + size_t GetAssetCount() const override { return assetCount_; }; + + bool DoAssetsOverlap(const PluginSortingInterface& plugin) const override { + const auto otherPlugin = dynamic_cast(&plugin); + return assetsOverlapWith.count(&plugin) != 0 || + otherPlugin->assetsOverlapWith.count(this) != 0; + } + + void AddOverlappingRecords(const PluginInterface& plugin) { + recordsOverlapWith.insert(&plugin); + } + + void SetOverrideRecordCount(size_t overrideRecordCount) { + overrideRecordCount_ = overrideRecordCount; + } + + void AddOverlappingAssets(const PluginSortingInterface& plugin) { + assetsOverlapWith.insert(&plugin); + } + + void SetAssetCount(size_t assetCount) { assetCount_ = assetCount; } + +private: + std::string name_; + std::set recordsOverlapWith; + std::set assetsOverlapWith; + size_t overrideRecordCount_{0}; + size_t assetCount_{0}; +}; +} + +class PluginGraphTest : public ::testing::Test { +public: + PluginSortingData CreatePluginSortingData(const std::string& name) { + const auto plugin = GetPlugin(name); + + return PluginSortingData( + plugin, PluginMetadata(), PluginMetadata(), {}, GameType::tes4, {}); + } + + plugingraph::TestPlugin* GetPlugin(const std::string& name) { + auto it = plugins.find(name); + + if (it != plugins.end()) { + return it->second.get(); + } + + const auto plugin = std::make_shared(name); + + return plugins.insert_or_assign(name, plugin).first->second.get(); + } + +private: + std::map> plugins; +}; + +TEST_F(PluginGraphTest, + topologicalSortWithNoLoadedPluginsShouldReturnAnEmptyList) { PluginGraph graph; std::vector sorted = graph.TopologicalSort(); EXPECT_TRUE(sorted.empty()); } + +TEST_F(PluginGraphTest, + addOverlapEdgesShouldNotAddEdgesBetweenNonOverlappingPlugins) { + PluginGraph graph; + + graph.AddVertex(CreatePluginSortingData("1.esp")); + graph.AddVertex(CreatePluginSortingData("2.esp")); + + const auto v1 = graph.GetVertexByName("1.esp").value(); + const auto v2 = graph.GetVertexByName("2.esp").value(); + + graph.AddOverlapEdges(); + + EXPECT_FALSE(graph.EdgeExists(v1, v2)); + EXPECT_FALSE(graph.EdgeExists(v2, v1)); +} + +TEST_F( + PluginGraphTest, + addOverlapEdgesShouldNotAddEdgeBetweenPluginsWithOverlappingRecordsAndEqualOverrideCounts) { + const auto p1 = GetPlugin("1.esp"); + const auto p2 = GetPlugin("2.esp"); + + p1->AddOverlappingRecords(*p2); + p1->SetOverrideRecordCount(1); + p2->SetOverrideRecordCount(1); + + PluginGraph graph; + + graph.AddVertex(CreatePluginSortingData("1.esp")); + graph.AddVertex(CreatePluginSortingData("2.esp")); + + const auto v1 = graph.GetVertexByName("1.esp").value(); + const auto v2 = graph.GetVertexByName("2.esp").value(); + + graph.AddOverlapEdges(); + + EXPECT_FALSE(graph.EdgeExists(v1, v2)); + EXPECT_FALSE(graph.EdgeExists(v2, v1)); +} + +TEST_F( + PluginGraphTest, + addOverlapEdgesShouldAddEdgeBetweenPluginsWithOverlappingRecordsAndInequalOverrideCounts) { + const auto p1 = GetPlugin("1.esp"); + const auto p2 = GetPlugin("2.esp"); + + p1->AddOverlappingRecords(*p2); + p1->SetOverrideRecordCount(2); + p2->SetOverrideRecordCount(1); + + PluginGraph graph; + + graph.AddVertex(CreatePluginSortingData("1.esp")); + graph.AddVertex(CreatePluginSortingData("2.esp")); + + const auto v1 = graph.GetVertexByName("1.esp").value(); + const auto v2 = graph.GetVertexByName("2.esp").value(); + + graph.AddOverlapEdges(); + + EXPECT_TRUE(graph.EdgeExists(v1, v2)); + EXPECT_FALSE(graph.EdgeExists(v2, v1)); +} + +TEST_F( + PluginGraphTest, + addOverlapEdgesShouldNotAddEdgeBetweenPluginsWithNonOverlappingRecordsAndInequalOverrideCounts) { + const auto p1 = GetPlugin("1.esp"); + const auto p2 = GetPlugin("2.esp"); + + p1->SetOverrideRecordCount(2); + p2->SetOverrideRecordCount(1); + + PluginGraph graph; + + graph.AddVertex(CreatePluginSortingData("1.esp")); + graph.AddVertex(CreatePluginSortingData("2.esp")); + + const auto v1 = graph.GetVertexByName("1.esp").value(); + const auto v2 = graph.GetVertexByName("2.esp").value(); + + graph.AddOverlapEdges(); + + EXPECT_FALSE(graph.EdgeExists(v1, v2)); + EXPECT_FALSE(graph.EdgeExists(v2, v1)); +} + +TEST_F( + PluginGraphTest, + addOverlapEdgesShouldNotAddEdgeBetweenPluginsWithAssetOverlapAndEqualAssetCounts) { + const auto p1 = GetPlugin("1.esp"); + const auto p2 = GetPlugin("2.esp"); + + p1->AddOverlappingAssets(*p2); + p1->SetAssetCount(1); + p2->SetAssetCount(1); + + PluginGraph graph; + + graph.AddVertex(CreatePluginSortingData("1.esp")); + graph.AddVertex(CreatePluginSortingData("2.esp")); + + const auto v1 = graph.GetVertexByName("1.esp").value(); + const auto v2 = graph.GetVertexByName("2.esp").value(); + + graph.AddOverlapEdges(); + + EXPECT_FALSE(graph.EdgeExists(v1, v2)); + EXPECT_FALSE(graph.EdgeExists(v2, v1)); +} + +TEST_F( + PluginGraphTest, + addOverlapEdgesShouldNotAddEdgeBetweenPluginsWithNoAssetOverlapAndInequalAssetCounts) { + const auto p1 = GetPlugin("1.esp"); + const auto p2 = GetPlugin("2.esp"); + + p1->SetAssetCount(2); + p2->SetAssetCount(1); + + PluginGraph graph; + + graph.AddVertex(CreatePluginSortingData("1.esp")); + graph.AddVertex(CreatePluginSortingData("2.esp")); + + const auto v1 = graph.GetVertexByName("1.esp").value(); + const auto v2 = graph.GetVertexByName("2.esp").value(); + + graph.AddOverlapEdges(); + + EXPECT_FALSE(graph.EdgeExists(v1, v2)); + EXPECT_FALSE(graph.EdgeExists(v2, v1)); +} + +TEST_F( + PluginGraphTest, + addOverlapEdgesShouldAddEdgeBetweenPluginsWithAssetOverlapAndInequalAssetCounts) { + const auto p1 = GetPlugin("1.esp"); + const auto p2 = GetPlugin("2.esp"); + + p1->AddOverlappingAssets(*p2); + p1->SetAssetCount(2); + p2->SetAssetCount(1); + + PluginGraph graph; + + graph.AddVertex(CreatePluginSortingData("1.esp")); + graph.AddVertex(CreatePluginSortingData("2.esp")); + + const auto v1 = graph.GetVertexByName("1.esp").value(); + const auto v2 = graph.GetVertexByName("2.esp").value(); + + graph.AddOverlapEdges(); + + EXPECT_TRUE(graph.EdgeExists(v1, v2)); + EXPECT_FALSE(graph.EdgeExists(v2, v1)); +} + +TEST_F( + PluginGraphTest, + addOverlapEdgesShouldCheckAssetsIfRecordsOverlapWithEqualOverrideCounts) { + const auto p1 = GetPlugin("1.esp"); + const auto p2 = GetPlugin("2.esp"); + + p1->AddOverlappingRecords(*p2); + p1->AddOverlappingAssets(*p2); + p1->SetAssetCount(2); + p2->SetAssetCount(1); + + PluginGraph graph; + + graph.AddVertex(CreatePluginSortingData("1.esp")); + graph.AddVertex(CreatePluginSortingData("2.esp")); + + const auto v1 = graph.GetVertexByName("1.esp").value(); + const auto v2 = graph.GetVertexByName("2.esp").value(); + + graph.AddOverlapEdges(); + + EXPECT_TRUE(graph.EdgeExists(v1, v2)); + EXPECT_FALSE(graph.EdgeExists(v2, v1)); +} + +TEST_F( + PluginGraphTest, + addOverlapEdgesShouldCheckAssetsIfRecordsDoNotOverlapWithInequalOverrideCounts) { + const auto p1 = GetPlugin("1.esp"); + const auto p2 = GetPlugin("2.esp"); + + p1->AddOverlappingAssets(*p2); + p1->SetAssetCount(2); + p2->SetAssetCount(1); + p1->SetOverrideRecordCount(1); + p2->SetOverrideRecordCount(2); + + PluginGraph graph; + + graph.AddVertex(CreatePluginSortingData("1.esp")); + graph.AddVertex(CreatePluginSortingData("2.esp")); + + const auto v1 = graph.GetVertexByName("1.esp").value(); + const auto v2 = graph.GetVertexByName("2.esp").value(); + + graph.AddOverlapEdges(); + + EXPECT_TRUE(graph.EdgeExists(v1, v2)); + EXPECT_FALSE(graph.EdgeExists(v2, v1)); +} + +TEST_F(PluginGraphTest, + addOverlapEdgesShouldChooseRecordOverlapOverAssetOverlap) { + const auto p1 = GetPlugin("1.esp"); + const auto p2 = GetPlugin("2.esp"); + + p1->AddOverlappingRecords(*p2); + p1->SetOverrideRecordCount(2); + p2->SetOverrideRecordCount(1); + p1->AddOverlappingAssets(*p2); + p1->SetAssetCount(1); + p2->SetAssetCount(2); + + PluginGraph graph; + + graph.AddVertex(CreatePluginSortingData("1.esp")); + graph.AddVertex(CreatePluginSortingData("2.esp")); + + const auto v1 = graph.GetVertexByName("1.esp").value(); + const auto v2 = graph.GetVertexByName("2.esp").value(); + + graph.AddOverlapEdges(); + + EXPECT_TRUE(graph.EdgeExists(v1, v2)); + EXPECT_FALSE(graph.EdgeExists(v2, v1)); +} } } diff --git a/src/tests/common_game_test_fixture.h b/src/tests/common_game_test_fixture.h index 0427981c..a94ff1d7 100644 --- a/src/tests/common_game_test_fixture.h +++ b/src/tests/common_game_test_fixture.h @@ -233,6 +233,18 @@ protected: return loadOrder; } + std::filesystem::path getSourcePluginsPath() const { + using std::filesystem::absolute; + if (GetParam() == GameType::tes3) + return absolute("./Morrowind/Data Files"); + else if (GetParam() == GameType::tes4) + return absolute("./Oblivion/Data"); + else if (GetParam() == GameType::fo4 || GetParam() == GameType::tes5se) + return absolute("./SkyrimSE/Data"); + else + return absolute("./Skyrim/Data"); + } + private: const std::filesystem::path rootTestPath; @@ -264,18 +276,6 @@ protected: const uint32_t blankEsmCrc; private: - std::filesystem::path getSourcePluginsPath() const { - using std::filesystem::absolute; - if (GetParam() == GameType::tes3) - return absolute("./Morrowind/Data Files"); - else if (GetParam() == GameType::tes4) - return absolute("./Oblivion/Data"); - else if (GetParam() == GameType::fo4 || GetParam() == GameType::tes5se) - return absolute("./SkyrimSE/Data"); - else - return absolute("./Skyrim/Data"); - } - inline std::string getMasterFile() const { if (GetParam() == GameType::tes3) return "Morrowind.esm";