Add support for masterlist prelude files

This allows common metadata to be shared across masterlists in a
'prelude' YAML file that is then injected into a masterlist when it is
loaded. Masterlist metadata can then reference anchors within the
prelude to make use of those common metadata.

The prelude file will be version controlled in a separate Git
repository, so the masterlist update functions also apply to managing
its versioning.

src/api/api_database.cpp

@@ -44,14 +44,26 @@ ApiDatabase::ApiDatabase(
 // Database Loading Functions
 ///////////////////////////////////
 
-void ApiDatabase::LoadLists(const std::filesystem::path& masterlistPath,
-                            const std::filesystem::path& userlistPath) {
+void ApiDatabase::LoadLists(
+    const std::filesystem::path& masterlistPath,
+    const std::filesystem::path& userlistPath,
+    const std::filesystem::path& masterlistPreludePath) {
   MetadataList temp;
   MetadataList userTemp;
 
   if (!masterlistPath.empty()) {
     if (std::filesystem::exists(masterlistPath)) {
-      temp.Load(masterlistPath);
+      if (!masterlistPreludePath.empty()) {
+        if (std::filesystem::exists(masterlistPreludePath)) {
+          temp.LoadWithPrelude(masterlistPath, masterlistPreludePath);
+        } else {
+          throw FileAccessError(
+              "The given masterlist prelude path does not exist: " +
+              masterlistPreludePath.u8string());
+        }
+      } else {
+        temp.Load(masterlistPath);
+      }
     } else {
       throw FileAccessError("The given masterlist path does not exist: " +
                             masterlistPath.u8string());
@@ -91,7 +103,7 @@ bool ApiDatabase::UpdateMasterlist(const std::filesystem::path& masterlistPath,
                                    const std::string& remoteURL,
                                    const std::string& remoteBranch) {
   if (!std::filesystem::is_directory(masterlistPath.parent_path()))
-    throw std::invalid_argument("Given masterlist path \"" +
+    throw std::invalid_argument("The path \"" +
                                 masterlistPath.u8string() +
                                 "\" does not have a valid parent directory.");
 

src/api/api_database.h

@@ -43,7 +43,8 @@ struct ApiDatabase : public DatabaseInterface {
   explicit ApiDatabase(std::shared_ptr<ConditionEvaluator> conditionEvaluator);
 
   void LoadLists(const std::filesystem::path& masterlist_path,
-                 const std::filesystem::path& userlist_path = "");
+                 const std::filesystem::path& userlist_path = "",
+                 const std::filesystem::path& masterlist_prelude_path = "");
 
   void WriteUserMetadata(const std::filesystem::path& outputFile,
                          const bool overwrite) const;

src/api/metadata_list.cpp

@@ -24,8 +24,10 @@
 
 #include "api/metadata_list.h"
 
+#include <boost/algorithm/string.hpp>
 #include <filesystem>
 #include <fstream>
+#include <sstream>
 
 #include "api/game/game.h"
 #include "api/helpers/logging.h"
@@ -36,6 +38,105 @@
 #include "loot/exception/file_access_error.h"
 
 namespace loot {
+constexpr std::string_view PRELUDE_ON_FIRST_LINE = "prelude:";
+constexpr std::string_view PRELUDE_ON_NEW_LINE = "\nprelude:";
+
+std::string read_to_string(const std::filesystem::path& filePath) {
+  std::ifstream in(filePath);
+  if (!in.good()) {
+    throw FileAccessError("Cannot open " + filePath.u8string());
+  }
+
+  auto content = std::string(std::istreambuf_iterator<char>(in),
+                             std::istreambuf_iterator<char>());
+
+  in.close();
+
+  return content;
+}
+
+std::optional<std::pair<size_t, size_t>> FindPreludeBounds(const std::string& masterlist) {
+  size_t startOfPrelude = std::string::npos;
+  size_t endOfPrelude = std::string::npos;
+
+  // This assumes that the metadata file is using block style at
+  // the top level, that ? indicators and tags are not used, and
+  // that key strings are unquoted.
+  if (boost::starts_with(masterlist, PRELUDE_ON_FIRST_LINE)) {
+    startOfPrelude = PRELUDE_ON_FIRST_LINE.size();
+  } else {
+    startOfPrelude = masterlist.find(PRELUDE_ON_NEW_LINE);
+
+    if (startOfPrelude != std::string::npos) {
+      // Skip the leading line break.
+      startOfPrelude += PRELUDE_ON_NEW_LINE.size();
+    }
+  }
+
+  if (startOfPrelude == std::string::npos) {
+    // No prelude to replace.
+    return std::nullopt;
+  }
+
+  // The end of the prelude is marked by a line break followed by a
+  // non-space, non-hash (#) character, as this means what follows is
+  // unindented content.
+  auto pos = startOfPrelude;
+  auto lastIndex = masterlist.size() - 1;
+  while (endOfPrelude == std::string::npos) {
+    auto nextLineBreakPos = masterlist.find("\n", pos);
+    if (nextLineBreakPos == std::string::npos ||
+        nextLineBreakPos == lastIndex) {
+      break;
+    }
+
+    pos = nextLineBreakPos + 1;
+
+    auto nextChar = masterlist[pos];
+    if (nextChar != ' ' && nextChar != '#' && nextChar != '\n') {
+      endOfPrelude = nextLineBreakPos;
+      break;
+    }
+  }
+
+  return std::make_pair(startOfPrelude, endOfPrelude);
+}
+
+// Indent all prelude content by two spaces to ensure it's parsed as part
+// of the prelude.
+std::string IndentPrelude(const std::string& prelude) {
+  auto newPrelude = "\n  " + boost::replace_all_copy(prelude, "\n", "\n  ");
+
+  boost::replace_all(newPrelude, "  \n", "\n");
+
+  if (boost::ends_with(newPrelude, "\n  ")) {
+    return newPrelude.substr(0, newPrelude.size() - 2);
+  }
+
+  return newPrelude;
+}
+
+std::string ReplaceMetadataListPrelude(const std::string& prelude,
+                                       const std::string& masterlist) {
+  auto preludeBounds = FindPreludeBounds(masterlist);
+
+  if (!preludeBounds.has_value()) {
+    // No prelude to replace.
+    return masterlist;
+  }
+
+  auto newPrelude = IndentPrelude(prelude);
+
+  auto [startOfPrelude, endOfPrelude] = preludeBounds.value();
+
+  if (endOfPrelude == std::string::npos) {
+    return masterlist.substr(0, startOfPrelude) + newPrelude;
+  }
+
+  return masterlist.substr(0, startOfPrelude) + newPrelude +
+         masterlist.substr(endOfPrelude);
+}
+
 void MetadataList::Load(const std::filesystem::path& filepath) {
   Clear();
 
@@ -48,12 +149,36 @@ void MetadataList::Load(const std::filesystem::path& filepath) {
   if (!in.good())
     throw FileAccessError("Cannot open " + filepath.u8string());
 
-  YAML::Node metadataList = YAML::Load(in);
+  this->Load(in, filepath);
+
   in.close();
+}
+
+void MetadataList::LoadWithPrelude(const std::filesystem::path& filePath,
+                                   const std::filesystem::path& preludePath) {
+  // Parsing YAML resolves references such that replacing the
+  // referenced keys entirely (rather than just replacing their values)
+  // does not cause aliases to be re-resolved, so the old values are
+  // retained.
+  // As such, replacing the prelude needs to happen before parsing,
+  // which means reading the files and performing string manipulation.
+  auto prelude_content = read_to_string(preludePath);
+  auto masterlist_content = read_to_string(filePath);
+
+  masterlist_content =
+      ReplaceMetadataListPrelude(prelude_content, masterlist_content);
+
+  auto stream = std::istringstream(masterlist_content);
+  this->Load(stream, filePath);
+}
+
+void MetadataList::Load(std::istream& istream,
+                        const std::filesystem::path& source_path) {
+  YAML::Node metadataList = YAML::Load(istream);
 
   if (!metadataList.IsMap())
     throw FileAccessError("The root of the metadata file " +
-                          filepath.u8string() + " is not a YAML map.");
+                          source_path.u8string() + " is not a YAML map.");
 
   if (metadataList["plugins"]) {
     for (const auto& node : metadataList["plugins"]) {
@@ -99,6 +224,7 @@ void MetadataList::Load(const std::filesystem::path& filepath) {
     groups_.insert(groups_.cbegin(), Group());
   }
 
+  auto logger = getLogger();
   if (logger) {
     logger->debug("File loaded successfully.");
   }
@@ -256,7 +382,8 @@ void MetadataList::EvalAllConditions(ConditionEvaluator& conditionEvaluator) {
     plugins_.clear();
 
   for (const auto& plugin : unevaluatedPlugins_) {
-    plugins_.emplace(plugin.first, conditionEvaluator.EvaluateAll(plugin.second));
+    plugins_.emplace(plugin.first,
+                     conditionEvaluator.EvaluateAll(plugin.second));
   }
 
   if (unevaluatedRegexPlugins_.empty())

src/api/metadata_list.h

@@ -46,9 +46,18 @@ struct hash<loot::Filename> {
 }
 
 namespace loot {
+
+// This assumes that the prelude and masterlist files both use
+// YAML's block style (at least up to the end of the prelude in the
+// latter). This is true for all official files.
+std::string ReplaceMetadataListPrelude(const std::string& prelude,
+                                       const std::string& masterlist);
+
 class MetadataList {
 public:
   void Load(const std::filesystem::path& filepath);
+  void LoadWithPrelude(const std::filesystem::path& filePath,
+                       const std::filesystem::path& preludePath);
   void Save(const std::filesystem::path& filepath) const;
   void Clear();
 
@@ -82,6 +91,8 @@ protected:
   std::unordered_map<Filename, PluginMetadata> unevaluatedPlugins_;
   std::vector<PluginMetadata> unevaluatedRegexPlugins_;
   std::vector<Message> unevaluatedMessages_;
+
+  void Load(std::istream& istream, const std::filesystem::path& source_path);
 };
 }
 

src/tests/api/interface/database_interface_test.h

@@ -26,7 +26,6 @@ along with LOOT.  If not, see
 #define LOOT_TESTS_API_INTERFACE_DATABASE_INTERFACE_TEST
 
 #include "loot/api.h"
-
 #include "tests/api/interface/api_game_operations_test.h"
 
 namespace loot {
@@ -124,13 +123,13 @@ protected:
 // Pass an empty first argument, as it's a prefix for the test instantation,
 // but we only have the one so no prefix is necessary.
 INSTANTIATE_TEST_SUITE_P(,
-                        DatabaseInterfaceTest,
-                        ::testing::Values(GameType::tes4,
-                                          GameType::tes5,
-                                          GameType::fo3,
-                                          GameType::fonv,
-                                          GameType::fo4,
-                                          GameType::tes5se));
+                         DatabaseInterfaceTest,
+                         ::testing::Values(GameType::tes4,
+                                           GameType::tes5,
+                                           GameType::fo3,
+                                           GameType::fonv,
+                                           GameType::fo4,
+                                           GameType::tes5se));
 
 TEST_P(DatabaseInterfaceTest,
        loadListsShouldSucceedEvenIfGameHandleIsDiscarded) {
@@ -169,6 +168,47 @@ TEST_P(DatabaseInterfaceTest,
   EXPECT_NO_THROW(db_->LoadLists(masterlistPath, userlistPath_));
 }
 
+TEST_P(
+    DatabaseInterfaceTest,
+    loadListsShouldThrowIfAMasterlistIsPresentButAPreludeDoesNotExistAtTheGivenPath) {
+  ASSERT_NO_THROW(GenerateMasterlist());
+
+  auto preludePath = localPath / "prelude.yaml";
+
+  EXPECT_THROW(db_->LoadLists(masterlistPath, "", preludePath),
+               FileAccessError);
+}
+
+TEST_P(DatabaseInterfaceTest,
+       loadListsShouldSucceedIfTheMasterlistAndPreludeAreBothPresent) {
+  using std::endl;
+
+  std::ofstream out(masterlistPath);
+  out << "prelude:" << endl
+      << "  - &ref" << endl
+      << "    type: say" << endl
+      << "    content: Loaded from same file" << endl
+      << "globals:" << endl
+      << "  - *ref" << endl;
+
+  out.close();
+
+  auto preludePath = localPath / "prelude.yaml";
+  out.open(preludePath);
+  out << "common:" << endl
+      << "  - &ref" << endl
+      << "    type: say" << endl
+      << "    content: Loaded from prelude" << endl;
+
+  EXPECT_NO_THROW(db_->LoadLists(masterlistPath, "", preludePath));
+
+  auto messages = db_->GetGeneralMessages();
+  ASSERT_EQ(1, messages.size());
+  EXPECT_EQ(MessageType::say, messages[0].GetType());
+  ASSERT_EQ(1, messages[0].GetContent().size());
+  EXPECT_EQ("Loaded from prelude", messages[0].GetContent()[0].GetText());
+}
+
 TEST_P(
     DatabaseInterfaceTest,
     writeUserMetadataShouldThrowIfTheFileAlreadyExistsAndTheOverwriteArgumentIsFalse) {
@@ -424,8 +464,7 @@ TEST_P(DatabaseInterfaceTest,
   EXPECT_TRUE(groups[1].GetAfterGroups().empty());
 
   EXPECT_EQ("group2", groups[2].GetName());
-  EXPECT_EQ(std::vector<std::string>({"group1"}),
-            groups[2].GetAfterGroups());
+  EXPECT_EQ(std::vector<std::string>({"group1"}), groups[2].GetAfterGroups());
 }
 
 TEST_P(
@@ -740,7 +779,8 @@ TEST_P(DatabaseInterfaceTest,
 
   std::vector<Message> expectedMessages({
       Message(MessageType::say,
-              generalMasterlistMessage, "file(\"" + missingEsp + "\")"),
+              generalMasterlistMessage,
+              "file(\"" + missingEsp + "\")"),
       Message(MessageType::say, generalUserlistMessage),
   });
   EXPECT_EQ(expectedMessages, messages);

src/tests/api/internals/metadata_list_test.h

@@ -26,7 +26,6 @@ along with LOOT.  If not, see
 #define LOOT_TESTS_API_INTERNALS_METADATA_LIST_TEST
 
 #include "api/metadata_list.h"
-
 #include "tests/common_game_test_fixture.h"
 
 namespace loot {
@@ -205,6 +204,38 @@ TEST_P(MetadataListTest,
   EXPECT_TRUE(metadataList.BashTags().empty());
 }
 
+TEST_P(
+    MetadataListTest,
+    loadWithPreludeShouldReplaceThePreludeInTheFirstFileWithTheContentOfTheSecond) {
+  using std::endl;
+
+  std::ofstream out(metadataPath);
+  out << "prelude:" << endl
+      << "  - &ref" << endl
+      << "    type: say" << endl
+      << "    content: Loaded from same file" << endl
+      << "globals:" << endl
+      << "  - *ref" << endl;
+
+  out.close();
+
+  auto preludePath = metadataFilesPath / "prelude.yaml";
+  out.open(preludePath);
+  out << "common:" << endl
+      << "  - &ref" << endl
+      << "    type: say" << endl
+      << "    content: Loaded from prelude" << endl;
+
+  MetadataList metadataList;
+  ASSERT_NO_THROW(metadataList.LoadWithPrelude(metadataPath, preludePath));
+
+  auto messages = metadataList.Messages();
+  ASSERT_EQ(1, messages.size());
+  EXPECT_EQ(MessageType::say, messages[0].GetType());
+  ASSERT_EQ(1, messages[0].GetContent().size());
+  EXPECT_EQ("Loaded from prelude", messages[0].GetContent()[0].GetText());
+}
+
 TEST_P(MetadataListTest, saveShouldWriteTheLoadedMetadataToTheGivenFilePath) {
   MetadataList metadataList;
   ASSERT_NO_THROW(metadataList.Load(metadataPath));
@@ -219,8 +250,9 @@ TEST_P(MetadataListTest, saveShouldWriteTheLoadedMetadataToTheGivenFilePath) {
   EXPECT_EQ(std::vector<std::string>({"C.Climate", "Relev"}),
             metadataList.BashTags());
 
-  auto expectedGroups =
-      std::vector<Group>({Group("default"), Group("group1", {"group2"}), Group("group2", {"default"})});
+  auto expectedGroups = std::vector<Group>({Group("default"),
+                                            Group("group1", {"group2"}),
+                                            Group("group2", {"default"})});
   EXPECT_EQ(expectedGroups, metadataList.Groups());
 
   EXPECT_EQ(std::vector<Message>({
@@ -391,6 +423,230 @@ TEST_P(
   EXPECT_EQ(blankEsp, plugin.GetName());
   EXPECT_TRUE(plugin.GetDirtyInfo().empty());
 }
+
+TEST(ReplaceMetadataListPrelude, shouldReturnAnEmptyStringIfGivenEmptyStrings) {
+  std::string prelude = "";
+  std::string masterlist = "";
+
+  auto result = ReplaceMetadataListPrelude(prelude, masterlist);
+
+  EXPECT_EQ(masterlist, result);
+}
+
+TEST(ReplaceMetadataListPrelude, shouldNotChangeAMasterlistWithNoPrelude) {
+  std::string prelude = R"(globals:
+  - type: note
+    content: A message.
+)";
+  std::string masterlist = R"(plugins:
+  - name: a.esp
+)";
+
+  auto result = ReplaceMetadataListPrelude(prelude, masterlist);
+
+  EXPECT_EQ(masterlist, result);
+}
+
+TEST(ReplaceMetadataListPrelude,
+     shouldReplaceAPreludeAtTheStartOfTheMasterlist) {
+  std::string prelude = R"(globals:
+  - type: note
+    content: A message.
+)";
+  std::string masterlist = R"(prelude:
+  a: b
+
+plugins:
+  - name: a.esp
+)";
+
+  auto result = ReplaceMetadataListPrelude(prelude, masterlist);
+
+  auto expectedResult = R"(prelude:
+  globals:
+    - type: note
+      content: A message.
+
+plugins:
+  - name: a.esp
+)";
+
+  EXPECT_EQ(expectedResult, result);
+}
+
+TEST(ReplaceMetadataListPrelude, shouldChangeAMasterlistThatEndsWithAPrelude) {
+  std::string prelude = R"(globals:
+  - type: note
+    content: A message.
+)";
+  std::string masterlist = R"(plugins:
+  - name: a.esp
+prelude:
+  a: b
+
+)";
+
+  auto result = ReplaceMetadataListPrelude(prelude, masterlist);
+
+  auto expectedResult = R"(plugins:
+  - name: a.esp
+prelude:
+  globals:
+    - type: note
+      content: A message.
+)";
+
+  EXPECT_EQ(expectedResult, result);
+}
+
+TEST(ReplaceMetadataListPrelude, shouldReplaceOnlyThePreludeInTheMasterlist) {
+  std::string prelude = R"(
+
+globals:
+  - type: note
+    content: A message.
+
+)";
+  std::string masterlist = R"(
+common:
+  key: value
+prelude:
+  a: b
+plugins:
+  - name: a.esp
+)";
+
+  auto result = ReplaceMetadataListPrelude(prelude, masterlist);
+
+  auto expectedResult = R"(
+common:
+  key: value
+prelude:
+
+
+  globals:
+    - type: note
+      content: A message.
+
+
+plugins:
+  - name: a.esp
+)";
+
+  EXPECT_EQ(expectedResult, result);
+}
+
+TEST(ReplaceMetadataListPrelude,
+     shouldSucceedIfGivenABlockStylePreludeAndABlockStyleMasterlist) {
+  std::string prelude = R"(globals:
+  - type: note
+    content: A message.
+)";
+  std::string masterlist = R"(prelude:
+  a: b
+
+plugins:
+  - name: a.esp
+)";
+
+  auto result = ReplaceMetadataListPrelude(prelude, masterlist);
+
+  auto expectedResult = R"(prelude:
+  globals:
+    - type: note
+      content: A message.
+
+plugins:
+  - name: a.esp
+)";
+
+  EXPECT_EQ(expectedResult, result);
+}
+
+TEST(ReplaceMetadataListPrelude,
+  shouldSucceedIfGivenAFlowStylePreludeAndABlockStyleMasterlist) {
+  std::string prelude = "globals: [{type: note, content: A message.}]";
+  std::string masterlist = R"(prelude:
+  a: b
+
+plugins:
+  - name: a.esp
+)";
+
+  auto result = ReplaceMetadataListPrelude(prelude, masterlist);
+
+  auto expectedResult = R"(prelude:
+  globals: [{type: note, content: A message.}]
+plugins:
+  - name: a.esp
+)";
+
+  EXPECT_EQ(expectedResult, result);
+}
+
+TEST(ReplaceMetadataListPrelude, doesNotChangeAFlowStyleMasterlist) {
+  std::string prelude = "globals: [{type: note, content: A message.}]";
+  std::string masterlist = "{prelude: {}, plugins: [{name: a.esp}]}";
+
+  auto result = ReplaceMetadataListPrelude(prelude, masterlist);
+
+  EXPECT_EQ(masterlist, result);
+}
+
+TEST(ReplaceMetadataListPrelude, shouldNotStopAtComments) {
+  std::string prelude = R"(globals:
+  - type: note
+    content: A message.
+)";
+  std::string masterlist = R"(prelude:
+  a: b
+# Comment line
+  c: d
+
+plugins:
+  - name: a.esp
+)";
+
+  auto result = ReplaceMetadataListPrelude(prelude, masterlist);
+
+  auto expectedResult = R"(prelude:
+  globals:
+    - type: note
+      content: A message.
+
+plugins:
+  - name: a.esp
+)";
+
+  EXPECT_EQ(expectedResult, result);
+}
+
+TEST(ReplaceMetadataListPrelude, shouldNotStopAtABlankLine) {
+  std::string prelude = R"(globals:
+  - type: note
+    content: A message.
+)";
+  std::string masterlist = R"(prelude:
+  a: b
+
+
+plugins:
+  - name: a.esp
+)";
+
+  auto result = ReplaceMetadataListPrelude(prelude, masterlist);
+
+  auto expectedResult = R"(prelude:
+  globals:
+    - type: note
+      content: A message.
+
+plugins:
+  - name: a.esp
+)";
+
+  EXPECT_EQ(expectedResult, result);
+}
 }
 }