Find predecessor group plugins on demand

Rather than storing them in PluginSortingData, which is now immutable.

This also means the predecessor groups plugins map can use vertices
instead of plugin names, which is a
little simpler.

docs/api/sorting.rst

@@ -80,11 +80,18 @@ plugins and the rest of the plugins in the graph.
 Group edges
 -----------
 
-Group-derived interdependencies are then evaluated. Each plugin's group-derived
-plugins are iterated over and individually checked to see if adding an edge from
-the group-derived plugin to the plugin would cause a cycle, and if not the edge
-is recorded. Once all potential edges have been checked, the recorded edges are
-added to the graph.
+For each plugin, the plugins that are members of groups that the current
+plugin's group loads after are iterated over and individually checked to see if
+adding an edge from the other group's plugin to the current plugin would cause a
+cycle. If not, the edge is queued for addition. If it would cause a cycle and
+one of the plugins is in the default group and the other group's plugin is
+master-flagged or the current plugin is not master flagged, then the plugin in
+the default group is recorded as one to skip adding edges to or from when the
+prospective edge involves any of the groups in the path from the other plugin
+to the current plugin.
+
+Once all the plugins have been iterated over, all the queued edges are added,
+skipping those edges identified in the earlier loop.
 
 At this point the plugin graph is checked for cycles, and an error is thrown if
 any are encountered, so that metadata (or indeed plugin data) that cause them

src/api/sorting/plugin_graph.cpp

@@ -34,6 +34,7 @@
 #include "api/helpers/text.h"
 #include "api/sorting/group_sort.h"
 #include "loot/exception/cyclic_interaction_error.h"
+#include "loot/exception/undefined_group_error.h"
 
 namespace loot {
 typedef boost::graph_traits<RawPluginGraph>::edge_descriptor edge_t;
@@ -80,6 +81,115 @@ private:
   std::vector<Vertex> trail;
 };
 
+struct GroupPlugin {
+  vertex_t vertex{0};
+  bool groupIsUserMetadata{false};
+};
+
+struct PredecessorGroupPlugin {
+  vertex_t vertex{0};
+  bool pathInvolvesUserMetadata{false};
+};
+
+std::unordered_map<std::string, std::vector<PredecessorGroupPlugin>>
+GetPredecessorGroupsPlugins(
+    const std::unordered_map<std::string, std::vector<GroupPlugin>>&
+        groupsPlugins,
+    const std::unordered_map<std::string, std::vector<PredecessorGroup>>&
+        predecessorGroupsMap) {
+  std::unordered_map<std::string, std::vector<PredecessorGroupPlugin>>
+      predecessorGroupsPlugins;
+  for (const auto& group : predecessorGroupsMap) {
+    std::vector<PredecessorGroupPlugin> predecessorGroupPlugins;
+
+    for (const auto& predecessorGroup : group.second) {
+      const auto pluginsIt = groupsPlugins.find(predecessorGroup.name);
+      if (pluginsIt != groupsPlugins.end()) {
+        // If the path from the predecessor group to this one involves user
+        // metadata, the plugins' paths all involve user metadata, otherwise
+        // only those plugins that belong to the predecessor group due to user
+        // metadata have a path involving user metadata.
+        for (const auto& groupPlugin : pluginsIt->second) {
+          predecessorGroupPlugins.push_back(PredecessorGroupPlugin{
+              groupPlugin.vertex,
+              predecessorGroup.pathInvolvesUserMetadata ||
+                  groupPlugin.groupIsUserMetadata});
+        }
+      }
+    }
+
+    predecessorGroupsPlugins.insert({group.first, predecessorGroupPlugins});
+  }
+
+  return predecessorGroupsPlugins;
+}
+
+std::unordered_map<std::string, std::vector<PredecessorGroupPlugin>>
+GetPredecessorGroupsPlugins(
+    const PluginGraph& graph,
+    const std::unordered_map<std::string, std::vector<PredecessorGroup>>&
+        predecessorGroupsMap) {
+  // Each element of the vector is a pair of a plugin name and if it's in the
+  // group due to user metadata.
+  std::unordered_map<std::string, std::vector<GroupPlugin>> groupsPlugins;
+
+  for (const vertex_t& vertex :
+       boost::make_iterator_range(graph.GetVertices())) {
+    const auto& plugin = graph.GetPlugin(vertex);
+    const auto groupName = plugin.GetGroup();
+    const auto groupPlugin = GroupPlugin{vertex, plugin.IsGroupUserMetadata()};
+
+    const auto groupIt = groupsPlugins.find(groupName);
+
+    if (groupIt == groupsPlugins.end()) {
+      groupsPlugins.emplace(groupName, std::vector<GroupPlugin>({groupPlugin}));
+    } else {
+      groupIt->second.push_back(groupPlugin);
+    }
+  }
+
+  // Sort plugins by their names. This is necessary to ensure that
+  // plugin precedessor group plugins are listed in a consistent
+  // order, which is important because that is the order in which
+  // group edges are added and differences could cause different
+  // sorting results.
+  for (auto& groupPlugins : groupsPlugins) {
+    std::sort(groupPlugins.second.begin(),
+              groupPlugins.second.end(),
+              [&](const GroupPlugin& lhs, const GroupPlugin& rhs) {
+                return graph.GetPlugin(lhs.vertex).GetName() <
+                       graph.GetPlugin(rhs.vertex).GetName();
+              });
+  }
+
+  // Map sets of transitive group dependencies to sets of transitive plugin
+  // dependencies.
+  return GetPredecessorGroupsPlugins(groupsPlugins, predecessorGroupsMap);
+}
+
+std::vector<PredecessorGroupPlugin> GetPredecessorGroupPlugins(
+    const PluginSortingData& plugin,
+    const std::unordered_map<std::string, std::vector<PredecessorGroupPlugin>>&
+        predecessorGroupsPlugins) {
+  const auto groupsIt = predecessorGroupsPlugins.find(plugin.GetGroup());
+  if (groupsIt == predecessorGroupsPlugins.end()) {
+    throw UndefinedGroupError(plugin.GetGroup());
+  }
+
+  if (plugin.IsGroupUserMetadata()) {
+    // If the current plugin is a member of its group due to user metadata,
+    // then all predecessor plugins are such due to user metadata.
+    auto predecessorGroupPlugins = groupsIt->second;
+    for (auto& predecessorGroupPlugin : predecessorGroupPlugins) {
+      predecessorGroupPlugin.pathInvolvesUserMetadata = true;
+    }
+
+    return predecessorGroupPlugins;
+  }
+
+  return groupsIt->second;
+}
+
 bool ShouldIgnorePlugin(
     const std::string& group,
     const std::string& pluginName,
@@ -459,16 +569,6 @@ std::optional<vertex_t> PluginGraph::GetVertexByName(
   return std::nullopt;
 }
 
-std::optional<vertex_t> PluginGraph::GetVertexByExactName(
-    const std::string& name) const {
-  const auto it = pluginNameVertexMap.find(name);
-  if (it != pluginNameVertexMap.end()) {
-    return it->second;
-  }
-
-  return std::nullopt;
-}
-
 const PluginSortingData& PluginGraph::GetPlugin(const vertex_t& vertex) const {
   return graph_[vertex];
 }
@@ -578,10 +678,7 @@ void PluginGraph::AddEdge(const vertex_t& fromVertex,
 }
 
 vertex_t PluginGraph::AddVertex(const PluginSortingData& plugin) {
-  const auto vertex = boost::add_vertex(plugin, graph_);
-  pluginNameVertexMap.emplace(plugin.GetName(), vertex);
-
-  return vertex;
+  return boost::add_vertex(plugin, graph_);
 }
 
 void PluginGraph::AddSpecificEdges() {
@@ -722,12 +819,17 @@ void PluginGraph::AddHardcodedPluginEdges(
 }
 
 void PluginGraph::AddGroupEdges(
-    const std::unordered_map<std::string, Group>& groups) {
+    const std::unordered_map<std::string, Group>& groups,
+    const std::unordered_map<std::string, std::vector<PredecessorGroup>>&
+        predecessorGroupsMap) {
   const auto logger = getLogger();
   if (logger) {
     logger->trace("Adding edges based on plugin group memberships...");
   }
 
+  const auto predecessorGroupsPlugins =
+      GetPredecessorGroupsPlugins(*this, predecessorGroupsMap);
+
   // Tuple fields are from, to, and edge type.
   std::vector<std::tuple<vertex_t, vertex_t, EdgeType>> acyclicEdges;
   std::map<std::string, std::unordered_set<std::string>> groupPluginsToIgnore;
@@ -735,16 +837,16 @@ void PluginGraph::AddGroupEdges(
   for (const vertex_t& vertex : boost::make_iterator_range(GetVertices())) {
     const auto& toPlugin = GetPlugin(vertex);
 
-    for (const auto& plugin : toPlugin.GetPredecessorGroupPlugins()) {
+    const auto predecessorGroupPlugins =
+        GetPredecessorGroupPlugins(toPlugin, predecessorGroupsPlugins);
+
+    for (const auto& plugin : predecessorGroupPlugins) {
       // After group plugin names are taken from other PluginSortingData names,
       // so exact string comparisons can be used.
-      const auto parentVertex = GetVertexByExactName(plugin.name);
-      if (!parentVertex.has_value()) {
-        continue;
-      }
+      const auto parentVertex = plugin.vertex;
 
-      if (PathExists(vertex, parentVertex.value())) {
-        const auto& fromPlugin = GetPlugin(parentVertex.value());
+      if (PathExists(vertex, parentVertex)) {
+        const auto& fromPlugin = GetPlugin(parentVertex);
 
         if (logger) {
           logger->debug(
@@ -793,8 +895,7 @@ void PluginGraph::AddGroupEdges(
                                 ? EdgeType::userGroup
                                 : EdgeType::masterlistGroup;
 
-      acyclicEdges.push_back(
-          std::make_tuple(parentVertex.value(), vertex, edgeType));
+      acyclicEdges.push_back(std::make_tuple(parentVertex, vertex, edgeType));
     }
   }
 

src/api/sorting/plugin_graph.h

@@ -33,6 +33,7 @@
 #include <boost/graph/graph_traits.hpp>
 #include <map>
 
+#include "api/sorting/group_sort.h"
 #include "api/sorting/plugin_sorting_data.h"
 #include "loot/enum/edge_type.h"
 #include "loot/metadata/group.h"
@@ -63,7 +64,6 @@ public:
   size_t CountVertices() const;
   std::pair<vertex_it, vertex_it> GetVertices() const;
   std::optional<vertex_t> GetVertexByName(const std::string& name) const;
-  std::optional<vertex_t> GetVertexByExactName(const std::string& name) const;
 
   const PluginSortingData& GetPlugin(const vertex_t& vertex) const;
 
@@ -93,14 +93,16 @@ public:
   void AddSpecificEdges();
   void AddHardcodedPluginEdges(
       const std::vector<std::string>& hardcodedPlugins);
-  void AddGroupEdges(const std::unordered_map<std::string, Group>& groups);
+  void AddGroupEdges(
+      const std::unordered_map<std::string, Group>& groups,
+      const std::unordered_map<std::string, std::vector<PredecessorGroup>>&
+          predecessorGroupsMap);
   void AddOverlapEdges();
   void AddTieBreakEdges();
 
 private:
   RawPluginGraph graph_;
   PathsCache pathsCache_;
-  std::map<std::string, vertex_t> pluginNameVertexMap;
 };
 }
 

src/api/sorting/plugin_sort.cpp

@@ -29,42 +29,8 @@
 #include "api/helpers/logging.h"
 #include "api/sorting/group_sort.h"
 #include "api/sorting/plugin_graph.h"
-#include "loot/exception/undefined_group_error.h"
 
 namespace loot {
-std::unordered_map<std::string, std::vector<PredecessorGroupPlugin>>
-GetPredecessorGroupPlugins(
-    const std::unordered_map<std::string,
-                             std::vector<std::pair<std::string, bool>>>&
-        groupPlugins,
-    const std::unordered_map<std::string, std::vector<PredecessorGroup>>&
-        predecessorGroupsMap) {
-  std::unordered_map<std::string, std::vector<PredecessorGroupPlugin>>
-      predecessorGroupsPlugins;
-  for (const auto& group : predecessorGroupsMap) {
-    std::vector<PredecessorGroupPlugin> predecessorGroupPlugins;
-
-    for (const auto& predecessorGroup : group.second) {
-      const auto pluginsIt = groupPlugins.find(predecessorGroup.name);
-      if (pluginsIt != groupPlugins.end()) {
-        // If the path from the predecessor group to this one involves user
-        // metadata, the plugins' paths all involve user metadata, otherwise
-        // only those plugins that belong to the predecessor group due to user
-        // metadata have a path involving user metadata.
-        for (const auto& groupPlugin : pluginsIt->second) {
-          predecessorGroupPlugins.push_back(PredecessorGroupPlugin{
-              groupPlugin.first,
-              predecessorGroup.pathInvolvesUserMetadata || groupPlugin.second});
-        }
-      }
-    }
-
-    predecessorGroupsPlugins.insert({group.first, predecessorGroupPlugins});
-  }
-
-  return predecessorGroupsPlugins;
-}
-
 int ComparePlugins(const PluginSortingData& plugin1,
                    const PluginSortingData& plugin2) {
   if (plugin1.GetLoadOrderIndex().has_value() &&
@@ -110,26 +76,7 @@ int ComparePlugins(const PluginSortingData& plugin1,
 std::vector<PluginSortingData> GetPluginsSortingData(
     const Game& game,
     const std::vector<std::string>& loadOrder) {
-  auto loadedPlugins = game.GetCache().GetPlugins();
-
-  // Sort plugins by their names. This is necessary to ensure that
-  // plugin precedessor group plugins are listed in a consistent
-  // order, which is important because that is the order in which
-  // group edges are added and differences could cause different
-  // sorting results.
-  std::sort(loadedPlugins.begin(),
-            loadedPlugins.end(),
-            [](const auto& lhs, const auto& rhs) {
-              if (!lhs) {
-                return false;
-              }
-
-              if (!rhs) {
-                return true;
-              }
-
-              return lhs->GetName() < rhs->GetName();
-            });
+  const auto loadedPlugins = game.GetCache().GetPlugins();
 
   std::vector<const PluginInterface*> loadedPluginInterfaces;
   std::transform(loadedPlugins.begin(),
@@ -164,64 +111,6 @@ std::vector<PluginSortingData> GetPluginsSortingData(
     pluginsSortingData.push_back(pluginSortingData);
   }
 
-  // Each element of the vector is a pair of a plugin name and if it's in the
-  // group due to user metadata.
-  std::unordered_map<std::string, std::vector<std::pair<std::string, bool>>>
-      groupPlugins;
-  for (const auto& plugin : pluginsSortingData) {
-    const auto groupName = plugin.GetGroup();
-    const auto groupPlugin =
-        std::make_pair(plugin.GetName(), plugin.IsGroupUserMetadata());
-    const auto groupIt = groupPlugins.find(groupName);
-
-    if (groupIt == groupPlugins.end()) {
-      groupPlugins.emplace(
-          groupName, std::vector<std::pair<std::string, bool>>({groupPlugin}));
-    } else {
-      groupIt->second.push_back(groupPlugin);
-    }
-  }
-
-  // Map sets of transitive group dependencies to sets of transitive plugin
-  // dependencies.
-  const auto predecessorGroupsMap = GetPredecessorGroups(
-      game.GetDatabase().GetGroups(false), game.GetDatabase().GetUserGroups());
-
-  // Replace the transitive after group names with the names of the plugins in
-  // those groups.
-  const auto predecessorGroupsPlugins =
-      GetPredecessorGroupPlugins(groupPlugins, predecessorGroupsMap);
-
-  // Add all transitive plugin dependencies for a group to the plugin's load
-  // after metadata.
-  const auto logger = getLogger();
-  for (auto& plugin : pluginsSortingData) {
-    if (logger) {
-      logger->trace(
-          "Plugin \"{}\" belongs to group \"{}\", setting after group plugins",
-          plugin.GetName(),
-          plugin.GetGroup());
-    }
-
-    const auto groupsIt = predecessorGroupsPlugins.find(plugin.GetGroup());
-    if (groupsIt == predecessorGroupsPlugins.end()) {
-      throw UndefinedGroupError(plugin.GetGroup());
-    }
-
-    if (plugin.IsGroupUserMetadata()) {
-      // If the current plugin is a member of its group due to user metadata,
-      // then all predecessor plugins are such due to user metadata.
-      auto predecessorGroupPlugins = groupsIt->second;
-      for (auto& predecessorGroupPlugin : predecessorGroupPlugins) {
-        predecessorGroupPlugin.pathInvolvesUserMetadata = true;
-      }
-
-      plugin.SetPredecessorGroupPlugins(predecessorGroupPlugins);
-    } else {
-      plugin.SetPredecessorGroupPlugins(groupsIt->second);
-    }
-  }
-
   // Sort the plugins according to into their existing load order, or
   // lexicographical ordering for pairs of plugins without load order positions.
   // This ensures a consistent iteration order for vertices given the same input
@@ -256,12 +145,14 @@ std::vector<std::string> GetPluginsWithHardcodedPositions(const Game& game) {
 std::vector<std::string> SortPluginGraph(
     PluginGraph& graph,
     const std::vector<std::string>& hardcodedPlugins,
-    const std::unordered_map<std::string, Group>& groupsMap) {
+    const std::unordered_map<std::string, Group>& groupsMap,
+    const std::unordered_map<std::string, std::vector<PredecessorGroup>>&
+        predecessorGroupsMap) {
   // Now add the interactions between plugins to the graph as edges.
   graph.AddSpecificEdges();
   graph.AddHardcodedPluginEdges(hardcodedPlugins);
 
-  graph.AddGroupEdges(groupsMap);
+  graph.AddGroupEdges(groupsMap, predecessorGroupsMap);
 
   // Check for cycles now because from this point on edges are only added if
   // they don't cause cycles, and adding tie-break edges is by far the slowest
@@ -342,10 +233,13 @@ std::vector<std::string> SortPlugins(
     groupsMap.emplace(group.GetName(), group);
   }
 
-  auto newLoadOrder =
-      SortPluginGraph(mastersGraph, hardcodedPlugins, groupsMap);
-  const auto newNonMastersLoadOrder =
-      SortPluginGraph(nonMastersGraph, hardcodedPlugins, groupsMap);
+  const auto predecessorGroupsMap = GetPredecessorGroups(
+      game.GetDatabase().GetGroups(false), game.GetDatabase().GetUserGroups());
+
+  auto newLoadOrder = SortPluginGraph(
+      mastersGraph, hardcodedPlugins, groupsMap, predecessorGroupsMap);
+  const auto newNonMastersLoadOrder = SortPluginGraph(
+      nonMastersGraph, hardcodedPlugins, groupsMap, predecessorGroupsMap);
 
   newLoadOrder.insert(newLoadOrder.end(),
                       newNonMastersLoadOrder.begin(),

src/api/sorting/plugin_sorting_data.cpp

@@ -147,16 +147,6 @@ bool PluginSortingData::IsGroupUserMetadata() const {
   return groupIsUserMetadata_;
 }
 
-std::vector<PredecessorGroupPlugin>
-PluginSortingData::GetPredecessorGroupPlugins() const {
-  return predecessorGroupPlugins_;
-}
-
-void PluginSortingData::SetPredecessorGroupPlugins(
-    std::vector<PredecessorGroupPlugin> plugins) {
-  predecessorGroupPlugins_ = plugins;
-}
-
 const std::vector<File>& PluginSortingData::GetMasterlistLoadAfterFiles()
     const {
   return masterlistLoadAfter_;

src/api/sorting/plugin_sorting_data.h

@@ -31,11 +31,6 @@
 #include "loot/metadata/plugin_metadata.h"
 
 namespace loot {
-struct PredecessorGroupPlugin {
-  std::string name;
-  bool pathInvolvesUserMetadata{false};
-};
-
 class PluginSortingData {
 public:
   explicit PluginSortingData() = default;
@@ -66,9 +61,6 @@ public:
   std::string GetGroup() const;
   bool IsGroupUserMetadata() const;
 
-  std::vector<PredecessorGroupPlugin> GetPredecessorGroupPlugins() const;
-  void SetPredecessorGroupPlugins(std::vector<PredecessorGroupPlugin> plugins);
-
   const std::vector<File>& GetMasterlistLoadAfterFiles() const;
   const std::vector<File>& GetUserLoadAfterFiles() const;
   const std::vector<File>& GetMasterlistRequirements() const;
@@ -79,7 +71,6 @@ public:
 private:
   const PluginSortingInterface* plugin_{nullptr};
   std::string group_;
-  std::vector<PredecessorGroupPlugin> predecessorGroupPlugins_;
 
   std::vector<File> masterlistLoadAfter_;
   std::vector<File> userLoadAfter_;