// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; using System.Xml; using EpicGames.Core; using Microsoft.Extensions.Logging; namespace EpicGames.BuildGraph { /// /// Options for how the graph should be printed /// public enum GraphPrintOptions { /// /// Includes a list of the graph options /// ShowCommandLineOptions = 0x1, /// /// Includes the list of dependencies for each node /// ShowDependencies = 0x2, /// /// Includes the list of notifiers for each node /// ShowNotifications = 0x4, } /// /// Definition of a graph. /// public class BgGraph { /// /// List of options, in the order they were specified /// public List Options { get; } = new List(); /// /// List of agents containing nodes to execute /// public List Agents { get; } = new List(); /// /// Mapping from name to agent /// public Dictionary NameToAgent { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// Mapping of names to the corresponding node. /// public Dictionary NameToNode { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// Mapping of names to the corresponding report. /// public Dictionary NameToReport { get; private set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// Mapping of names to their corresponding node output. /// public Dictionary TagNameToNodeOutput { get; private set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// Mapping of aggregate names to their respective nodes /// public Dictionary NameToAggregate { get; private set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// List of badges that can be displayed for this build /// public List Badges { get; } = new List(); /// /// List of labels that can be displayed for this build /// public List Labels { get; } = new List(); /// /// Diagnostic messages for this graph /// public List Diagnostics { get; } = new List(); /// /// Default constructor /// public BgGraph() { } /// /// Checks whether a given name already exists /// /// The name to check. /// True if the name exists, false otherwise. public bool ContainsName(string name) { return NameToNode.ContainsKey(name) || NameToReport.ContainsKey(name) || NameToAggregate.ContainsKey(name); } /// /// Tries to resolve the given name to one or more nodes. Checks for aggregates, and actual nodes. /// /// The name to search for /// If the name is a match, receives an array of nodes and their output names /// True if the name was found, false otherwise. public bool TryResolveReference(string name, [NotNullWhen(true)] out BgNode[]? outNodes) { // Check if it's a tag reference or node reference if (name.StartsWith("#")) { // Check if it's a regular node or output name BgNodeOutput? output; if (TagNameToNodeOutput.TryGetValue(name, out output)) { outNodes = new BgNode[] { output.ProducingNode }; return true; } } else { // Check if it's a regular node or output name BgNode? node; if (NameToNode.TryGetValue(name, out node)) { outNodes = new BgNode[] { node }; return true; } // Check if it's an aggregate name BgAggregate? aggregate; if (NameToAggregate.TryGetValue(name, out aggregate)) { outNodes = aggregate.RequiredNodes.ToArray(); return true; } // Check if it's a group name BgAgent? agent; if (NameToAgent.TryGetValue(name, out agent)) { outNodes = agent.Nodes.ToArray(); return true; } } // Otherwise fail outNodes = null; return false; } /// /// Tries to resolve the given name to one or more node outputs. Checks for aggregates, and actual nodes. /// /// The name to search for /// If the name is a match, receives an array of nodes and their output names /// True if the name was found, false otherwise. public bool TryResolveInputReference(string name, [NotNullWhen(true)] out BgNodeOutput[]? outOutputs) { // Check if it's a tag reference or node reference if (name.StartsWith("#")) { // Check if it's a regular node or output name BgNodeOutput? output; if (TagNameToNodeOutput.TryGetValue(name, out output)) { outOutputs = new BgNodeOutput[] { output }; return true; } } else { // Check if it's a regular node or output name BgNode? node; if (NameToNode.TryGetValue(name, out node)) { outOutputs = node.Outputs.Union(node.Inputs).ToArray(); return true; } // Check if it's an aggregate name BgAggregate? aggregate; if (NameToAggregate.TryGetValue(name, out aggregate)) { outOutputs = aggregate.RequiredNodes.SelectMany(x => x.Outputs.Union(x.Inputs)).Distinct().ToArray(); return true; } } // Otherwise fail outOutputs = null; return false; } static void AddDependencies(BgNode node, HashSet retainNodes) { if (retainNodes.Add(node)) { foreach (BgNode inputDependency in node.InputDependencies) { AddDependencies(inputDependency, retainNodes); } } } /// /// Cull the graph to only include the given nodes and their dependencies /// /// A set of target nodes to build public void Select(IEnumerable targetNodes) { // Find this node and all its dependencies HashSet retainNodes = new HashSet(); foreach (BgNode targetNode in targetNodes) { AddDependencies(targetNode, retainNodes); } // Remove all the nodes which are not marked to be kept foreach (BgAgent agent in Agents) { agent.Nodes = agent.Nodes.Where(x => retainNodes.Contains(x)).ToList(); } // Remove all the empty agents Agents.RemoveAll(x => x.Nodes.Count == 0); // Trim down the list of nodes for each report to the ones that are being built foreach (BgReport report in NameToReport.Values) { report.Nodes.RemoveWhere(x => !retainNodes.Contains(x)); } // Remove all the empty reports NameToReport = NameToReport.Where(x => x.Value.Nodes.Count > 0).ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.InvariantCultureIgnoreCase); // Remove all the order dependencies which are no longer part of the graph. Since we don't need to build them, we don't need to wait for them foreach (BgNode node in retainNodes) { node.OrderDependencies = node.OrderDependencies.Where(x => retainNodes.Contains(x)).ToArray(); } // Create a new list of aggregates for everything that's left Dictionary newNameToAggregate = new Dictionary(NameToAggregate.Comparer); foreach (BgAggregate aggregate in NameToAggregate.Values) { if (aggregate.RequiredNodes.All(x => retainNodes.Contains(x))) { newNameToAggregate[aggregate.Name] = aggregate; } } NameToAggregate = newNameToAggregate; // Remove any labels that are no longer value foreach (BgLabel label in Labels) { label.RequiredNodes.RemoveWhere(x => !retainNodes.Contains(x)); label.IncludedNodes.RemoveWhere(x => !retainNodes.Contains(x)); } Labels.RemoveAll(x => x.RequiredNodes.Count == 0); // Remove any badges which do not have all their dependencies Badges.RemoveAll(x => x.Nodes.Any(y => !retainNodes.Contains(y))); // Remove any diagnostics which are no longer part of the graph Diagnostics.RemoveAll(x => (x.EnclosingNode != null && !retainNodes.Contains(x.EnclosingNode)) || (x.EnclosingAgent != null && !Agents.Contains(x.EnclosingAgent))); } /// /// Writes a preprocessed build graph to a script file /// /// The file to load /// Schema file for validation public void Write(FileReference file, FileReference schemaFile) { XmlWriterSettings settings = new XmlWriterSettings(); settings.Indent = true; settings.IndentChars = "\t"; using (XmlWriter writer = XmlWriter.Create(file.FullName, settings)) { writer.WriteStartElement("BuildGraph", "http://www.epicgames.com/BuildGraph"); if (schemaFile != null) { writer.WriteAttributeString("schemaLocation", "http://www.w3.org/2001/XMLSchema-instance", "http://www.epicgames.com/BuildGraph " + schemaFile.MakeRelativeTo(file.Directory)); } foreach (BgAgent agent in Agents) { agent.Write(writer); } foreach (BgAggregate aggregate in NameToAggregate.Values) { // If the aggregate has no required elements, skip it. if (aggregate.RequiredNodes.Count == 0) { continue; } writer.WriteStartElement("Aggregate"); writer.WriteAttributeString("Name", aggregate.Name); writer.WriteAttributeString("Requires", String.Join(";", aggregate.RequiredNodes.Select(x => x.Name))); writer.WriteEndElement(); } foreach (BgLabel label in Labels) { writer.WriteStartElement("Label"); if (label.DashboardCategory != null) { writer.WriteAttributeString("Category", label.DashboardCategory); } writer.WriteAttributeString("Name", label.DashboardName); writer.WriteAttributeString("Requires", String.Join(";", label.RequiredNodes.Select(x => x.Name))); HashSet includedNodes = new HashSet(label.IncludedNodes); includedNodes.ExceptWith(label.IncludedNodes.SelectMany(x => x.InputDependencies)); includedNodes.ExceptWith(label.RequiredNodes); if (includedNodes.Count > 0) { writer.WriteAttributeString("Include", String.Join(";", includedNodes.Select(x => x.Name))); } HashSet excludedNodes = new HashSet(label.IncludedNodes); excludedNodes.UnionWith(label.IncludedNodes.SelectMany(x => x.InputDependencies)); excludedNodes.ExceptWith(label.IncludedNodes); excludedNodes.ExceptWith(excludedNodes.ToArray().SelectMany(x => x.InputDependencies)); if (excludedNodes.Count > 0) { writer.WriteAttributeString("Exclude", String.Join(";", excludedNodes.Select(x => x.Name))); } writer.WriteEndElement(); } foreach (BgReport report in NameToReport.Values) { writer.WriteStartElement("Report"); writer.WriteAttributeString("Name", report.Name); writer.WriteAttributeString("Requires", String.Join(";", report.Nodes.Select(x => x.Name))); writer.WriteEndElement(); } foreach (BgBadge badge in Badges) { writer.WriteStartElement("Badge"); writer.WriteAttributeString("Name", badge.Name); if (badge.Project != null) { writer.WriteAttributeString("Project", badge.Project); } if (badge.Change != 0) { writer.WriteAttributeString("Change", badge.Change.ToString()); } writer.WriteAttributeString("Requires", String.Join(";", badge.Nodes.Select(x => x.Name))); writer.WriteEndElement(); } writer.WriteEndElement(); } } /// /// Export the build graph to a Json file, for parallel execution by the build system /// /// Output file to write /// Set of nodes which have been completed public void Export(FileReference file, HashSet completedNodes) { // Find all the nodes which we're actually going to execute. We'll use this to filter the graph. HashSet nodesToExecute = new HashSet(); foreach (BgNode node in Agents.SelectMany(x => x.Nodes)) { if (!completedNodes.Contains(node)) { nodesToExecute.Add(node); } } // Open the output file using (JsonWriter jsonWriter = new JsonWriter(file.FullName)) { jsonWriter.WriteObjectStart(); // Write all the agents jsonWriter.WriteArrayStart("Groups"); foreach (BgAgent agent in Agents) { BgNode[] nodes = agent.Nodes.Where(x => nodesToExecute.Contains(x)).ToArray(); if (nodes.Length > 0) { jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", agent.Name); jsonWriter.WriteArrayStart("Agent Types"); foreach (string agentType in agent.PossibleTypes) { jsonWriter.WriteValue(agentType); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Nodes"); foreach (BgNode node in nodes) { jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", node.Name); jsonWriter.WriteValue("DependsOn", String.Join(";", node.GetDirectOrderDependencies().Where(x => nodesToExecute.Contains(x)))); jsonWriter.WriteValue("RunEarly", node.RunEarly); jsonWriter.WriteObjectStart("Notify"); jsonWriter.WriteValue("Default", String.Join(";", node.NotifyUsers)); jsonWriter.WriteValue("Submitters", String.Join(";", node.NotifySubmitters)); jsonWriter.WriteValue("Warnings", node.NotifyOnWarnings); jsonWriter.WriteObjectEnd(); jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } } jsonWriter.WriteArrayEnd(); // Write all the badges jsonWriter.WriteArrayStart("Badges"); foreach (BgBadge badge in Badges) { BgNode[] dependencies = badge.Nodes.Where(x => nodesToExecute.Contains(x)).ToArray(); if (dependencies.Length > 0) { // Reduce that list to the smallest subset of direct dependencies HashSet directDependencies = new HashSet(dependencies); foreach (BgNode dependency in dependencies) { directDependencies.ExceptWith(dependency.OrderDependencies); } jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", badge.Name); if (!String.IsNullOrEmpty(badge.Project)) { jsonWriter.WriteValue("Project", badge.Project); } if (badge.Change != 0) { jsonWriter.WriteValue("Change", badge.Change); } jsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => dependencies.Contains(x)).Select(x => x.Name))); jsonWriter.WriteValue("DirectDependencies", String.Join(";", directDependencies.Select(x => x.Name))); jsonWriter.WriteObjectEnd(); } } jsonWriter.WriteArrayEnd(); // Write all the triggers and reports. jsonWriter.WriteArrayStart("Reports"); foreach (BgReport report in NameToReport.Values) { BgNode[] dependencies = report.Nodes.Where(x => nodesToExecute.Contains(x)).ToArray(); if (dependencies.Length > 0) { // Reduce that list to the smallest subset of direct dependencies HashSet directDependencies = new HashSet(dependencies); foreach (BgNode dependency in dependencies) { directDependencies.ExceptWith(dependency.OrderDependencies); } jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", report.Name); jsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => dependencies.Contains(x)).Select(x => x.Name))); jsonWriter.WriteValue("DirectDependencies", String.Join(";", directDependencies.Select(x => x.Name))); jsonWriter.WriteValue("Notify", String.Join(";", report.NotifyUsers)); jsonWriter.WriteValue("IsTrigger", false); jsonWriter.WriteObjectEnd(); } } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } } /// /// Export the build graph to a Json file for parsing by Horde /// /// Output file to write public void ExportForHorde(FileReference file) { DirectoryReference.CreateDirectory(file.Directory); using (JsonWriter jsonWriter = new JsonWriter(file.FullName)) { jsonWriter.WriteObjectStart(); jsonWriter.WriteArrayStart("Groups"); foreach (BgAgent agent in Agents) { jsonWriter.WriteObjectStart(); jsonWriter.WriteArrayStart("Types"); foreach (string possibleType in agent.PossibleTypes) { jsonWriter.WriteValue(possibleType); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Nodes"); foreach (BgNode node in agent.Nodes) { jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", node.Name); jsonWriter.WriteValue("RunEarly", node.RunEarly); jsonWriter.WriteValue("Warnings", node.NotifyOnWarnings); jsonWriter.WriteArrayStart("InputDependencies"); foreach (string inputDependency in node.GetDirectInputDependencies().Select(x => x.Name)) { jsonWriter.WriteValue(inputDependency); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("OrderDependencies"); foreach (string orderDependency in node.GetDirectOrderDependencies().Select(x => x.Name)) { jsonWriter.WriteValue(orderDependency); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectStart("Annotations"); foreach ((string key, string value) in node.Annotations) { jsonWriter.WriteValue(key, value); } jsonWriter.WriteObjectEnd(); jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Aggregates"); foreach (BgAggregate aggregate in NameToAggregate.Values) { jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", aggregate.Name); jsonWriter.WriteArrayStart("Nodes"); foreach (BgNode requiredNode in aggregate.RequiredNodes.OrderBy(x => x.Name)) { jsonWriter.WriteValue(requiredNode.Name); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Labels"); foreach (BgLabel label in Labels) { jsonWriter.WriteObjectStart(); if (!String.IsNullOrEmpty(label.DashboardName)) { jsonWriter.WriteValue("Name", label.DashboardName); } if (!String.IsNullOrEmpty(label.DashboardCategory)) { jsonWriter.WriteValue("Category", label.DashboardCategory); } if (!String.IsNullOrEmpty(label.UgsBadge)) { jsonWriter.WriteValue("UgsBadge", label.UgsBadge); } if (!String.IsNullOrEmpty(label.UgsProject)) { jsonWriter.WriteValue("UgsProject", label.UgsProject); } if (label.Change != BgLabelChange.Current) { jsonWriter.WriteValue("Change", label.Change.ToString()); } jsonWriter.WriteArrayStart("RequiredNodes"); foreach (BgNode requiredNode in label.RequiredNodes.OrderBy(x => x.Name)) { jsonWriter.WriteValue(requiredNode.Name); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("IncludedNodes"); foreach (BgNode includedNode in label.IncludedNodes.OrderBy(x => x.Name)) { jsonWriter.WriteValue(includedNode.Name); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Badges"); foreach (BgBadge badge in Badges) { HashSet dependencies = badge.Nodes; if (dependencies.Count > 0) { // Reduce that list to the smallest subset of direct dependencies HashSet directDependencies = new HashSet(dependencies); foreach (BgNode dependency in dependencies) { directDependencies.ExceptWith(dependency.OrderDependencies); } jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", badge.Name); if (!String.IsNullOrEmpty(badge.Project)) { jsonWriter.WriteValue("Project", badge.Project); } if (badge.Change != 0) { jsonWriter.WriteValue("Change", badge.Change); } jsonWriter.WriteValue("Dependencies", String.Join(";", directDependencies.Select(x => x.Name))); jsonWriter.WriteObjectEnd(); } } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } } /// /// Print the contents of the graph /// /// Set of nodes which are already complete /// Options for how to print the graph /// public void Print(HashSet completedNodes, GraphPrintOptions printOptions, ILogger logger) { // Print the options if ((printOptions & GraphPrintOptions.ShowCommandLineOptions) != 0) { // Get the list of messages List> parameters = new List>(); foreach (BgOption option in Options) { string name = String.Format("-set:{0}=...", option.Name); StringBuilder description = new StringBuilder(option.Description); if (!String.IsNullOrEmpty(option.DefaultValue)) { description.AppendFormat(" (Default: {0})", option.DefaultValue); } parameters.Add(new KeyValuePair(name, description.ToString())); } // Format them to the log if (parameters.Count > 0) { logger.LogInformation(""); logger.LogInformation("Options:"); logger.LogInformation(""); List lines = new List(); HelpUtils.FormatTable(parameters, 4, 24, HelpUtils.WindowWidth - 20, lines); foreach (string line in lines) { logger.Log(LogLevel.Information, line); } } } // Output all the triggers in order logger.LogInformation(""); logger.LogInformation("Graph:"); foreach (BgAgent agent in Agents) { logger.LogInformation(" Agent: {0} ({1})", agent.Name, String.Join(";", agent.PossibleTypes)); foreach (BgNode node in agent.Nodes) { logger.LogInformation(" Node: {0}{1}", node.Name, completedNodes.Contains(node) ? " (completed)" : node.RunEarly ? " (early)" : ""); if (printOptions.HasFlag(GraphPrintOptions.ShowDependencies)) { HashSet inputDependencies = new HashSet(node.GetDirectInputDependencies()); foreach (BgNode inputDependency in inputDependencies) { logger.LogInformation(" input> {0}", inputDependency.Name); } HashSet orderDependencies = new HashSet(node.GetDirectOrderDependencies()); foreach (BgNode orderDependency in orderDependencies.Except(inputDependencies)) { logger.LogInformation(" after> {0}", orderDependency.Name); } } if (printOptions.HasFlag(GraphPrintOptions.ShowNotifications)) { string label = node.NotifyOnWarnings ? "warnings" : "errors"; foreach (string user in node.NotifyUsers) { logger.LogInformation(" {0}> {1}", label, user); } foreach (string submitter in node.NotifySubmitters) { logger.LogInformation(" {0}> submitters to {1}", label, submitter); } } } } logger.LogInformation(""); // Print out all the non-empty aggregates BgAggregate[] aggregates = NameToAggregate.Values.OrderBy(x => x.Name).ToArray(); if (aggregates.Length > 0) { logger.LogInformation("Aggregates:"); foreach (string aggregateName in aggregates.Select(x => x.Name)) { logger.LogInformation(" {0}", aggregateName); } logger.LogInformation(""); } } } }