Files
UnrealEngineUWP/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/CsCompileTask.cs
Matthew Griffin 2adcad66d3 Make sure we don't try to sign third party dlls
Check whether referenced pdbs marked as copy local exist in the output dir instead of where they originally were - avoids adding files that don't exist when using enumerate only option
#lockdown Nick.Penwarden
#rb Ben.Marsh
#codereview Ben.Marsh

[CL 3057599 by Matthew Griffin in Main branch]
2016-07-20 10:14:01 -04:00

672 lines
24 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
using UnrealBuildTool;
namespace AutomationTool.Tasks
{
/// <summary>
/// Parameters for a task that compiles a C# project
/// </summary>
public class CsCompileTaskParameters
{
/// <summary>
/// The C# project file to be compile. More than one project file can be specified by separating with semicolons.
/// </summary>
[TaskParameter]
public string Project;
/// <summary>
/// The configuration to compile
/// </summary>
[TaskParameter(Optional = true)]
public string Configuration;
/// <summary>
/// The platform to compile
/// </summary>
[TaskParameter(Optional = true)]
public string Platform;
/// <summary>
/// Additional options to pass to the compiler
/// </summary>
[TaskParameter(Optional = true)]
public string Arguments;
/// <summary>
/// Directory containing output files
/// </summary>
[TaskParameter(Optional = true)]
public string OutputDir;
/// <summary>
/// Patterns for output files
/// </summary>
[TaskParameter(Optional = true)]
public string OutputFiles;
/// <summary>
/// Only enumerate build products; do not actually compile the projects.
/// </summary>
[TaskParameter(Optional = true)]
public bool EnumerateOnly;
/// <summary>
/// Tag to be applied to build products of this task
/// </summary>
[TaskParameter(Optional = true, ValidationType = TaskParameterValidationType.TagList)]
public string Tag;
/// <summary>
/// Tag to be applied to any non-private references the projects have
/// (i.e. those that are external and not copied into the output dir)
/// </summary>
[TaskParameter(Optional = true, ValidationType = TaskParameterValidationType.TagList)]
public string TagReferences;
}
/// <summary>
/// Compile a C# project file
/// </summary>
[TaskElement("CsCompile", typeof(CsCompileTaskParameters))]
public class CsCompileTask : CustomTask
{
/// <summary>
/// Parameters for the task
/// </summary>
CsCompileTaskParameters Parameters;
/// <summary>
/// Constructor.
/// </summary>
/// <param name="InParameters">Parameters for this task</param>
public CsCompileTask(CsCompileTaskParameters InParameters)
{
Parameters = InParameters;
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet<FileReference> BuildProducts, Dictionary<string, HashSet<FileReference>> TagNameToFileSet)
{
// Get the project file
HashSet<FileReference> ProjectFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Project, TagNameToFileSet);
foreach(FileReference ProjectFile in ProjectFiles)
{
if(!ProjectFile.Exists())
{
CommandUtils.LogError("Couldn't find project file '{0}'", ProjectFile.FullName);
return false;
}
if(!ProjectFile.HasExtension(".csproj"))
{
CommandUtils.LogError("File '{0}' is not a C# project", ProjectFile.FullName);
return false;
}
}
// Get the default properties
Dictionary<string, string> Properties = new Dictionary<string,string>(StringComparer.InvariantCultureIgnoreCase);
if(!String.IsNullOrEmpty(Parameters.Platform))
{
Properties["Platform"] = Parameters.Platform;
}
if(!String.IsNullOrEmpty(Parameters.Configuration))
{
Properties["Configuration"] = Parameters.Configuration;
}
// Build the arguments and run the build
if(!Parameters.EnumerateOnly)
{
List<string> Arguments = new List<string>();
foreach(KeyValuePair<string, string> PropertyPair in Properties)
{
Arguments.Add(String.Format("/property:{0}={1}", CommandUtils.MakePathSafeToUseWithCommandLine(PropertyPair.Key), CommandUtils.MakePathSafeToUseWithCommandLine(PropertyPair.Value)));
}
if(!String.IsNullOrEmpty(Parameters.Arguments))
{
Arguments.Add(Parameters.Arguments);
}
Arguments.Add("/verbosity:minimal");
Arguments.Add("/nologo");
foreach(FileReference ProjectFile in ProjectFiles)
{
CommandUtils.MsBuild(CommandUtils.CmdEnv, ProjectFile.FullName, String.Join(" ", Arguments), null);
}
}
// Try to figure out the output files
HashSet<FileReference> ProjectBuildProducts;
HashSet<FileReference> ProjectReferences;
if(!FindBuildProducts(ProjectFiles, Properties, out ProjectBuildProducts, out ProjectReferences))
{
return false;
}
// Apply the optional tag to the produced archive
foreach(string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(ProjectBuildProducts);
}
if (!String.IsNullOrEmpty(Parameters.TagReferences))
{
// Apply the optional tag to any references
foreach (string TagName in FindTagNamesFromList(Parameters.TagReferences))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(ProjectReferences);
}
}
// Merge them into the standard set of build products
BuildProducts.UnionWith(ProjectBuildProducts);
return true;
}
/// <summary>
/// Output this task out to an XML writer.
/// </summary>
public override void Write(XmlWriter Writer)
{
Write(Writer, Parameters);
}
/// <summary>
/// Find all the tags which are used as inputs to this task
/// </summary>
/// <returns>The tag names which are read by this task</returns>
public override IEnumerable<string> FindConsumedTagNames()
{
return FindTagNamesFromFilespec(Parameters.Project);
}
/// <summary>
/// Find all the tags which are modified by this task
/// </summary>
/// <returns>The tag names which are modified by this task</returns>
public override IEnumerable<string> FindProducedTagNames()
{
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
yield return TagName;
}
foreach (string TagName in FindTagNamesFromList(Parameters.TagReferences))
{
yield return TagName;
}
}
/// <summary>
/// Find all the build products created by compiling the given project file
/// </summary>
/// <param name="ProjectFiles">Initial project file to read. All referenced projects will also be read.</param>
/// <param name="InitialProperties">Mapping of property name to value</param>
/// <param name="OutBuildProducts">Receives a set of build products on success</param>
/// <param name="OutReferences">Receives a set of non-private references on success</param>
/// <returns>True if the build products were found, false otherwise.</returns>
static bool FindBuildProducts(HashSet<FileReference> ProjectFiles, Dictionary<string, string> InitialProperties, out HashSet<FileReference> OutBuildProducts, out HashSet<FileReference> OutReferences)
{
// Read all the project information into a dictionary
Dictionary<FileReference, CsProjectInfo> FileToProjectInfo = new Dictionary<FileReference,CsProjectInfo>();
foreach(FileReference ProjectFile in ProjectFiles)
{
if(!ReadProjectsRecursively(ProjectFile, InitialProperties, FileToProjectInfo))
{
OutBuildProducts = null;
OutReferences = null;
return false;
}
}
// Find all the build products and references
HashSet<FileReference> BuildProducts = new HashSet<FileReference>();
HashSet<FileReference> References = new HashSet<FileReference>();
foreach(KeyValuePair<FileReference, CsProjectInfo> Pair in FileToProjectInfo)
{
CsProjectInfo ProjectInfo = Pair.Value;
// Add the standard build products
DirectoryReference OutputDir = ProjectInfo.GetOutputDir(Pair.Key.Directory);
ProjectInfo.AddBuildProducts(OutputDir, BuildProducts);
// Add the referenced assemblies
foreach(KeyValuePair<FileReference, bool> Reference in ProjectInfo.References)
{
FileReference OtherAssembly = Reference.Key;
if (Reference.Value)
{
// Add reference from the output dir
FileReference OutputFile = FileReference.Combine(OutputDir, OtherAssembly.GetFileName());
BuildProducts.Add(OutputFile);
FileReference OutputSymbolFile = OutputFile.ChangeExtension(".pdb");
if(OutputSymbolFile.Exists())
{
BuildProducts.Add(OutputSymbolFile);
}
}
else
{
// Add reference directly
References.Add(OtherAssembly);
FileReference SymbolFile = OtherAssembly.ChangeExtension(".pdb");
if(SymbolFile.Exists())
{
References.Add(SymbolFile);
}
}
}
// Add build products from all the referenced projects. MSBuild only copy the directly referenced build products, not recursive references or other assemblies.
foreach(CsProjectInfo OtherProjectInfo in ProjectInfo.ProjectReferences.Where(x => x.Value).Select(x => FileToProjectInfo[x.Key]))
{
OtherProjectInfo.AddBuildProducts(OutputDir, BuildProducts);
}
}
// Update the output set
OutBuildProducts = BuildProducts;
OutReferences = References;
return true;
}
/// <summary>
/// Read a project file, plus all the project files it references.
/// </summary>
/// <param name="File">Project file to read</param>
/// <param name="InitialProperties">Mapping of property name to value for the initial project</param>
/// <param name="FileToProjectInfo"></param>
/// <returns>True if the projects were read correctly, false (and prints an error to the log) if not</returns>
static bool ReadProjectsRecursively(FileReference File, Dictionary<string, string> InitialProperties, Dictionary<FileReference, CsProjectInfo> FileToProjectInfo)
{
// Early out if we've already read this project, return succes
if(FileToProjectInfo.ContainsKey(File))
{
return true;
}
// Try to read this project
CsProjectInfo ProjectInfo;
if(!CsProjectInfo.TryRead(File, InitialProperties, out ProjectInfo))
{
CommandUtils.LogError("Couldn't read project '{0}'", File.FullName);
return false;
}
// Add it to the project lookup, and try to read all the projects it references
FileToProjectInfo.Add(File, ProjectInfo);
return ProjectInfo.ProjectReferences.Keys.All(x => ReadProjectsRecursively(x, new Dictionary<string, string>(), FileToProjectInfo));
}
}
/// <summary>
/// Basic information from a preprocessed C# project file. Supports reading a project file, expanding simple conditions in it, parsing property values, assembly references and references to other projects.
/// </summary>
class CsProjectInfo
{
/// <summary>
/// Evaluated properties from the project file
/// </summary>
public Dictionary<string, string> Properties;
/// <summary>
/// Mapping of referenced assemblies to their 'CopyLocal' (aka 'Private') setting.
/// </summary>
public Dictionary<FileReference, bool> References = new Dictionary<FileReference,bool>();
/// <summary>
/// Mapping of referenced projects to their 'CopyLocal' (aka 'Private') setting.
/// </summary>
public Dictionary<FileReference, bool> ProjectReferences = new Dictionary<FileReference,bool>();
/// <summary>
/// Constructor
/// </summary>
/// <param name="InProperties">Initial mapping of property names to values</param>
CsProjectInfo(Dictionary<string, string> InProperties)
{
Properties = new Dictionary<string,string>(InProperties);
}
/// <summary>
/// Resolve the project's output directory
/// </summary>
/// <param name="BaseDirectory">Base directory to resolve relative paths to</param>
/// <returns>The configured output directory</returns>
public DirectoryReference GetOutputDir(DirectoryReference BaseDirectory)
{
string OutputPath;
if(Properties.TryGetValue("OutputPath", out OutputPath))
{
return DirectoryReference.Combine(BaseDirectory, OutputPath);
}
else
{
return BaseDirectory;
}
}
/// <summary>
/// Adds build products from the project to the given set.
/// </summary>
/// <param name="OutputDir">Output directory for the build products. May be different to the project's output directory in the case that we're copying local to another project.</param>
/// <param name="BuildProducts">Set to receive the list of build products</param>
public bool AddBuildProducts(DirectoryReference OutputDir, HashSet<FileReference> BuildProducts)
{
string OutputType, AssemblyName;
if(Properties.TryGetValue("OutputType", out OutputType) && Properties.TryGetValue("AssemblyName", out AssemblyName))
{
switch(OutputType)
{
case "Exe":
BuildProducts.Add(FileReference.Combine(OutputDir, AssemblyName + ".exe"));
BuildProducts.Add(FileReference.Combine(OutputDir, AssemblyName + ".pdb"));
FileReference ExeConfig = FileReference.Combine(OutputDir, AssemblyName + ".exe.config");
if (ExeConfig.Exists()) { BuildProducts.Add(ExeConfig); }
FileReference ExeMDB = FileReference.Combine(OutputDir, AssemblyName + "exe.mdb");
if (ExeMDB.Exists()) { BuildProducts.Add(ExeMDB); }
return true;
case "Library":
BuildProducts.Add(FileReference.Combine(OutputDir, AssemblyName + ".dll"));
BuildProducts.Add(FileReference.Combine(OutputDir, AssemblyName + ".pdb"));
return true;
}
}
return false;
}
/// <summary>
/// Attempts to read project information for the given file.
/// </summary>
/// <param name="File">The project file to read</param>
/// <param name="Properties">Initial set of property values</param>
/// <param name="OutProjectInfo">If successful, the parsed project info</param>
/// <returns>True if the project was read successfully, false otherwise</returns>
public static bool TryRead(FileReference File, Dictionary<string, string> Properties, out CsProjectInfo OutProjectInfo)
{
// Read the project file
XmlDocument Document = new XmlDocument();
Document.Load(File.FullName);
// Check the root element is the right type
// HashSet<FileReference> ProjectBuildProducts = new HashSet<FileReference>();
if(Document.DocumentElement.Name != "Project")
{
OutProjectInfo = null;
return false;
}
// Parse the basic structure of the document, updating properties and recursing into other referenced projects as we go
CsProjectInfo ProjectInfo = new CsProjectInfo(Properties);
foreach(XmlElement Element in Document.DocumentElement.ChildNodes.OfType<XmlElement>())
{
switch(Element.Name)
{
case "PropertyGroup":
if(EvaluateCondition(Element, ProjectInfo.Properties))
{
ParsePropertyGroup(Element, ProjectInfo.Properties);
}
break;
case "ItemGroup":
if(EvaluateCondition(Element, ProjectInfo.Properties))
{
ParseItemGroup(File.Directory, Element, ProjectInfo);
}
break;
}
}
// Return the complete project
OutProjectInfo = ProjectInfo;
return true;
}
/// <summary>
/// Parses a 'PropertyGroup' element.
/// </summary>
/// <param name="ParentElement">The parent 'PropertyGroup' element</param>
/// <param name="Properties">Dictionary mapping property names to values</param>
static void ParsePropertyGroup(XmlElement ParentElement, Dictionary<string, string> Properties)
{
// We need to know the overridden output type and output path for the selected configuration.
foreach(XmlElement Element in ParentElement.ChildNodes.OfType<XmlElement>())
{
if(EvaluateCondition(Element, Properties))
{
Properties[Element.Name] = ExpandProperties(Element.InnerText, Properties);
}
}
}
/// <summary>
/// Parses an 'ItemGroup' element.
/// </summary>
/// <param name="BaseDirectory">Base directory to resolve relative paths against</param>
/// <param name="ParentElement">The parent 'ItemGroup' element</param>
/// <param name="ProjectInfo">Project info object to be updated</param>
static void ParseItemGroup(DirectoryReference BaseDirectory, XmlElement ParentElement, CsProjectInfo ProjectInfo)
{
// Parse any external assembly references
foreach(XmlElement ItemElement in ParentElement.ChildNodes.OfType<XmlElement>())
{
switch(ItemElement.Name)
{
case "Reference":
// Reference to an external assembly
if(EvaluateCondition(ItemElement, ProjectInfo.Properties))
{
ParseReference(BaseDirectory, ItemElement, ProjectInfo.References);
}
break;
case "ProjectReference":
// Reference to another project
if(EvaluateCondition(ItemElement, ProjectInfo.Properties))
{
ParseProjectReference(BaseDirectory, ItemElement, ProjectInfo.ProjectReferences);
}
break;
}
}
}
/// <summary>
/// Parses an assembly reference from a given 'Reference' element
/// </summary>
/// <param name="BaseDirectory">Directory to resolve relative paths against</param>
/// <param name="ParentElement">The parent 'Reference' element</param>
/// <param name="ProjectReferences">Dictionary of project files to a bool indicating whether the assembly should be copied locally to the referencing project.</param>
static void ParseReference(DirectoryReference BaseDirectory, XmlElement ParentElement, Dictionary<FileReference, bool> References)
{
string HintPath = GetChildElementString(ParentElement, "HintPath", null);
if(!String.IsNullOrEmpty(HintPath))
{
FileReference AssemblyFile = FileReference.Combine(BaseDirectory, HintPath);
bool bPrivate = GetChildElementBoolean(ParentElement, "Private", true);
References.Add(AssemblyFile, bPrivate);
}
}
/// <summary>
/// Parses a project reference from a given 'ProjectReference' element
/// </summary>
/// <param name="BaseDirectory">Directory to resolve relative paths against</param>
/// <param name="ParentElement">The parent 'ProjectReference' element</param>
/// <param name="ProjectReferences">Dictionary of project files to a bool indicating whether the outputs of the project should be copied locally to the referencing project.</param>
static void ParseProjectReference(DirectoryReference BaseDirectory, XmlElement ParentElement, Dictionary<FileReference, bool> ProjectReferences)
{
string IncludePath = ParentElement.GetAttribute("Include");
if(!String.IsNullOrEmpty(IncludePath))
{
FileReference ProjectFile = FileReference.Combine(BaseDirectory, IncludePath);
bool bPrivate = GetChildElementBoolean(ParentElement, "Private", true);
ProjectReferences[ProjectFile] = bPrivate;
}
}
/// <summary>
/// Reads the inner text of a child XML element
/// </summary>
/// <param name="ParentElement">The parent element to check</param>
/// <param name="Name">Name of the child element</param>
/// <param name="DefaultValue">Default value to return if the child element is missing</param>
/// <returns>The contents of the child element, or default value if it's not present</returns>
static string GetChildElementString(XmlElement ParentElement, string Name, string DefaultValue)
{
XmlElement ChildElement = ParentElement.ChildNodes.OfType<XmlElement>().FirstOrDefault(x => x.Name == Name);
if(ChildElement == null)
{
return DefaultValue;
}
else
{
return ChildElement.InnerText ?? DefaultValue;
}
}
/// <summary>
/// Read a child XML element with the given name, and parse it as a boolean.
/// </summary>
/// <param name="ParentElement">Parent element to check</param>
/// <param name="Name">Name of the child element to look for</param>
/// <param name="DefaultValue">Default value to return if the element is missing or not a valid bool</param>
/// <returns>The parsed boolean, or the default value</returns>
static bool GetChildElementBoolean(XmlElement ParentElement, string Name, bool DefaultValue)
{
string Value = GetChildElementString(ParentElement, Name, null);
if(Value == null)
{
return DefaultValue;
}
else if(Value.Equals("True", StringComparison.InvariantCultureIgnoreCase))
{
return true;
}
else if(Value.Equals("False", StringComparison.InvariantCultureIgnoreCase))
{
return false;
}
else
{
return DefaultValue;
}
}
/// <summary>
/// Evaluate whether the optional MSBuild condition on an XML element evaluates to true. Currently only supports 'ABC' == 'DEF' style expressions, but can be expanded as needed.
/// </summary>
/// <param name="Element">The XML element to check</param>
/// <param name="Properties">Dictionary mapping from property names to values.</param>
/// <returns></returns>
static bool EvaluateCondition(XmlElement Element, Dictionary<string, string> Properties)
{
// Read the condition attribute. If it's not present, assume it evaluates to true.
string Condition = Element.GetAttribute("Condition");
if(String.IsNullOrEmpty(Condition))
{
return true;
}
// Expand all the properties
Condition = ExpandProperties(Condition, Properties);
// Tokenize the condition
string[] Tokens = Tokenize(Condition);
// Try to evaluate it. We only support a very limited class of condition expressions at the moment, but it's enough to parse standard projects
bool bResult;
if(Tokens.Length == 3 && Tokens[0].StartsWith("'") && Tokens[1] == "==" && Tokens[2].StartsWith("'"))
{
bResult = String.Compare(Tokens[0], Tokens[2], StringComparison.InvariantCultureIgnoreCase) == 0;
}
else
{
throw new AutomationException("Couldn't parse condition in project file");
}
return bResult;
}
/// <summary>
/// Expand MSBuild properties within a string. If referenced properties are not in this dictionary, the process' environment variables are expanded. Unknown properties are expanded to an empty string.
/// </summary>
/// <param name="Text">The input string to expand</param>
/// <param name="Properties">Dictionary mapping from property names to values.</param>
/// <returns>String with all properties expanded.</returns>
static string ExpandProperties(string Text, Dictionary<string, string> Properties)
{
string NewText = Text;
for (int Idx = NewText.IndexOf("$("); Idx != -1; Idx = NewText.IndexOf("$(", Idx))
{
// Find the end of the variable name
int EndIdx = NewText.IndexOf(')', Idx + 2);
if (EndIdx != -1)
{
// Extract the variable name from the string
string Name = NewText.Substring(Idx + 2, EndIdx - (Idx + 2));
// Find the value for it, either from the dictionary or the environment block
string Value;
if(!Properties.TryGetValue(Name, out Value))
{
Value = Environment.GetEnvironmentVariable(Name) ?? "";
}
// Replace the variable, or skip past it
NewText = NewText.Substring(0, Idx) + Value + NewText.Substring(EndIdx + 1);
// Make sure we skip over the expanded variable; we don't want to recurse on it.
Idx += Value.Length;
}
}
return NewText;
}
/// <summary>
/// Split an MSBuild condition into tokens
/// </summary>
/// <param name="Condition">The condition expression</param>
/// <returns>Array of the parsed tokens</returns>
static string[] Tokenize(string Condition)
{
List<string> Tokens = new List<string>();
for(int Idx = 0; Idx < Condition.Length; Idx++)
{
if(Idx + 1 < Condition.Length && Condition[Idx] == '=' && Condition[Idx + 1] == '=')
{
// "==" operator
Idx++;
Tokens.Add("==");
}
else if(Condition[Idx] == '\'')
{
// Quoted string
int StartIdx = Idx++;
while(Idx + 1 < Condition.Length && Condition[Idx] != '\'')
{
Idx++;
}
Tokens.Add(Condition.Substring(StartIdx, Idx - StartIdx));
}
else if(!Char.IsWhiteSpace(Condition[Idx]))
{
// Other token; assume a single character.
string Token = Condition.Substring(Idx, 1);
Tokens.Add(Token);
}
}
return Tokens.ToArray();
}
}
}