// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. using System; using System.IO; using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Diagnostics; using System.Runtime.Serialization; using Tools.DotNETCommon; using System.Collections.Concurrent; namespace UnrealBuildTool { /// /// Class which compiles (and caches) rules assemblies for different folders. /// public class RulesCompiler { /// /// Enum for types of rules files. Should match extensions in RulesFileExtensions. /// public enum RulesFileType { /// /// .build.cs files /// Module, /// /// .target.cs files /// Target, /// /// .automation.csproj files /// AutomationModule } /// /// Cached list of rules files in each directory of each type /// class RulesFileCache { public List ModuleRules = new List(); public List TargetRules = new List(); public List AutomationModules = new List(); } /// Map of root folders to a cached list of all UBT-related source files in that folder or any of its sub-folders. /// We cache these file names so we can avoid searching for them later on. static Dictionary RootFolderToRulesFileCache = new Dictionary(); /// /// /// #if NET_CORE const string FrameworkAssemblyExtension = "_NetCore.dll"; #else const string FrameworkAssemblyExtension = ".dll"; #endif /// /// /// /// /// /// /// /// /// /// public static List FindAllRulesSourceFiles(RulesFileType RulesFileType, List GameFolders, List ForeignPlugins, List AdditionalSearchPaths, bool bIncludeEngine = true, bool bIncludeEnterprise = true) { List Folders = new List(); // Add all engine source (including third party source) if (bIncludeEngine) { Folders.Add(UnrealBuildTool.EngineSourceDirectory); } if(bIncludeEnterprise) { Folders.Add(UnrealBuildTool.EnterpriseSourceDirectory); } // @todo plugin: Disallow modules from including plugin modules as dependency modules? (except when the module is part of that plugin) // Get all the root folders for plugins List RootFolders = new List(); if (bIncludeEngine) { RootFolders.Add(UnrealBuildTool.EngineDirectory); } if(bIncludeEnterprise) { RootFolders.Add(UnrealBuildTool.EnterpriseDirectory); } if (GameFolders != null) { RootFolders.AddRange(GameFolders); } // Find all the plugin source directories foreach (DirectoryReference RootFolder in RootFolders) { DirectoryReference PluginsFolder = DirectoryReference.Combine(RootFolder, "Plugins"); foreach (FileReference PluginFile in Plugins.EnumeratePlugins(PluginsFolder)) { Folders.Add(DirectoryReference.Combine(PluginFile.Directory, "Source")); } } // Add all the extra plugin folders if (ForeignPlugins != null) { foreach (FileReference ForeignPlugin in ForeignPlugins) { Folders.Add(DirectoryReference.Combine(ForeignPlugin.Directory, "Source")); } } // Add in the game folders to search if (GameFolders != null) { foreach (DirectoryReference GameFolder in GameFolders) { DirectoryReference GameSourceFolder = DirectoryReference.Combine(GameFolder, "Source"); Folders.Add(GameSourceFolder); DirectoryReference GameIntermediateSourceFolder = DirectoryReference.Combine(GameFolder, "Intermediate", "Source"); Folders.Add(GameIntermediateSourceFolder); } } // Process the additional search path, if sent in if (AdditionalSearchPaths != null) { foreach (DirectoryReference AdditionalSearchPath in AdditionalSearchPaths) { if (AdditionalSearchPath != null) { if (DirectoryReference.Exists(AdditionalSearchPath)) { Folders.Add(AdditionalSearchPath); } else { throw new BuildException("Couldn't find AdditionalSearchPath for rules source files '{0}'", AdditionalSearchPath); } } } } // Iterate over all the folders to check List SourceFiles = new List(); HashSet UniqueSourceFiles = new HashSet(); foreach (DirectoryReference Folder in Folders) { IReadOnlyList SourceFilesForFolder = FindAllRulesFiles(Folder, RulesFileType); foreach (FileReference SourceFile in SourceFilesForFolder) { if (UniqueSourceFiles.Add(SourceFile)) { SourceFiles.Add(SourceFile); } } } return SourceFiles; } /// /// Invalidate the cache for the givcen directory /// /// Directory to invalidate public static void InvalidateRulesFileCache(string DirectoryPath) { DirectoryReference Directory = new DirectoryReference(DirectoryPath); RootFolderToRulesFileCache.Remove(Directory); DirectoryLookupCache.InvalidateCachedDirectory(Directory); } /// /// Prefetch multiple directories in parallel /// /// The directories to cache private static void PrefetchRulesFiles(IEnumerable Directories) { ThreadPoolWorkQueue Queue = null; try { foreach(DirectoryReference Directory in Directories) { if(!RootFolderToRulesFileCache.ContainsKey(Directory)) { RulesFileCache Cache = new RulesFileCache(); RootFolderToRulesFileCache[Directory] = Cache; if(Queue == null) { Queue = new ThreadPoolWorkQueue(); } DirectoryItem DirectoryItem = DirectoryItem.GetItemByDirectoryReference(Directory); Queue.Enqueue(() => FindAllRulesFilesRecursively(DirectoryItem, Cache, Queue)); } } } finally { if(Queue != null) { Queue.Dispose(); Queue = null; } } } /// /// Finds all the rules of the given type under a given directory /// /// Directory to search /// Type of rules to return /// List of rules files of the given type private static IReadOnlyList FindAllRulesFiles(DirectoryReference Directory, RulesFileType Type) { // Check to see if we've already cached source files for this folder RulesFileCache Cache; if (!RootFolderToRulesFileCache.TryGetValue(Directory, out Cache)) { Cache = new RulesFileCache(); using(ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue()) { DirectoryItem BaseDirectory = DirectoryItem.GetItemByDirectoryReference(Directory); Queue.Enqueue(() => FindAllRulesFilesRecursively(BaseDirectory, Cache, Queue)); } RootFolderToRulesFileCache[Directory] = Cache; } // Get the list of files of the type we're looking for if (Type == RulesCompiler.RulesFileType.Module) { return Cache.ModuleRules; } else if (Type == RulesCompiler.RulesFileType.Target) { return Cache.TargetRules; } else if (Type == RulesCompiler.RulesFileType.AutomationModule) { return Cache.AutomationModules; } else { throw new BuildException("Unhandled rules type: {0}", Type); } } /// /// Search through a directory tree for any rules files /// /// The root directory to search from /// Receives all the discovered rules files /// Queue for adding additional tasks to private static void FindAllRulesFilesRecursively(DirectoryItem Directory, RulesFileCache Cache, ThreadPoolWorkQueue Queue) { // Scan all the files in this directory bool bSearchSubFolders = true; foreach (FileItem File in Directory.EnumerateFiles()) { if (File.HasExtension(".build.cs")) { lock(Cache.ModuleRules) { Cache.ModuleRules.Add(File.Location); } bSearchSubFolders = false; } else if (File.HasExtension(".target.cs")) { lock(Cache.TargetRules) { Cache.TargetRules.Add(File.Location); } } else if (File.HasExtension(".automation.csproj")) { lock(Cache.AutomationModules) { Cache.AutomationModules.Add(File.Location); } bSearchSubFolders = false; } } // If we didn't find anything to stop the search, search all the subdirectories too if (bSearchSubFolders) { foreach (DirectoryItem SubDirectory in Directory.EnumerateDirectories()) { Queue.Enqueue(() => FindAllRulesFilesRecursively(SubDirectory, Cache, Queue)); } } } /// /// The cached rules assembly for engine modules and targets. /// private static RulesAssembly EngineRulesAssembly; /// /// The cached rules assembly for enterprise modules and targets. /// private static RulesAssembly EnterpriseRulesAssembly; /// /// Map of assembly names we've already compiled and loaded to their Assembly and list of game folders. This is used to prevent /// trying to recompile the same assembly when ping-ponging between different types of targets /// private static Dictionary LoadedAssemblyMap = new Dictionary(); /// /// Creates the engine rules assembly /// /// Whether to use a precompiled engine /// Whether to skip compilation for this assembly /// New rules assembly public static RulesAssembly CreateEngineRulesAssembly(bool bUsePrecompiled, bool bSkipCompile) { if (EngineRulesAssembly == null) { IReadOnlyList IncludedPlugins = Plugins.ReadEnginePlugins(UnrealBuildTool.EngineDirectory); EngineRulesAssembly = CreateEngineOrEnterpriseRulesAssembly(UnrealBuildTool.EngineDirectory, ProjectFileGenerator.EngineProjectFileNameBase, IncludedPlugins, UnrealBuildTool.IsEngineInstalled() || bUsePrecompiled, bSkipCompile, null); } return EngineRulesAssembly; } /// /// Creates the enterprise rules assembly /// /// Whether to use a precompiled enterprise and engine folder /// Whether to skip compilation for this assembly /// New rules assembly. Returns null if the enterprise directory is unavailable. public static RulesAssembly CreateEnterpriseRulesAssembly(bool bUsePrecompiled, bool bSkipCompile) { if (EnterpriseRulesAssembly == null) { if (DirectoryReference.Exists(UnrealBuildTool.EnterpriseDirectory)) { IReadOnlyList IncludedPlugins = Plugins.ReadEnterprisePlugins(UnrealBuildTool.EnterpriseDirectory); EnterpriseRulesAssembly = CreateEngineOrEnterpriseRulesAssembly(UnrealBuildTool.EnterpriseDirectory, ProjectFileGenerator.EnterpriseProjectFileNameBase, IncludedPlugins, UnrealBuildTool.IsEnterpriseInstalled() || bUsePrecompiled, bSkipCompile, CreateEngineRulesAssembly(bUsePrecompiled, bSkipCompile)); } else { Log.TraceWarning("Trying to build an enterprise target but the enterprise directory is missing. Falling back on engine components only."); // If we're asked for the enterprise rules assembly but the enterprise directory is missing, fallback on the engine rules assembly return CreateEngineRulesAssembly(bUsePrecompiled, bSkipCompile); } } return EnterpriseRulesAssembly; } /// /// Creates a rules assembly /// /// The root directory to create rules for /// A prefix for the assembly file name /// List of plugins to include in this assembly /// Whether the assembly should be marked as installed /// Whether to skip compilation for this assembly /// The parent rules assembly /// New rules assembly private static RulesAssembly CreateEngineOrEnterpriseRulesAssembly(DirectoryReference RootDirectory, string AssemblyPrefix, IReadOnlyList Plugins, bool bReadOnly, bool bSkipCompile, RulesAssembly Parent) { DirectoryReference SourceDirectory = DirectoryReference.Combine(RootDirectory, "Source"); DirectoryReference ProgramsDirectory = DirectoryReference.Combine(SourceDirectory, "Programs"); // Find the shared modules, excluding the programs directory. These are used to create an assembly with the bContainsEngineModules flag set to true. List EngineModuleFiles = new List(); using(Timeline.ScopeEvent("Finding engine modules")) { foreach (DirectoryReference SubDirectory in DirectoryLookupCache.EnumerateDirectories(SourceDirectory)) { if(SubDirectory != ProgramsDirectory) { EngineModuleFiles.AddRange(FindAllRulesFiles(SubDirectory, RulesFileType.Module)); } } } // Add all the plugin modules too Dictionary ModuleFileToPluginInfo = new Dictionary(); using(Timeline.ScopeEvent("Finding plugin modules")) { FindModuleRulesForPlugins(Plugins, EngineModuleFiles, ModuleFileToPluginInfo); } // Create the assembly FileReference EngineAssemblyFileName = FileReference.Combine(RootDirectory, "Intermediate", "Build", "BuildRules", AssemblyPrefix + "Rules" + FrameworkAssemblyExtension); RulesAssembly EngineAssembly = new RulesAssembly(RootDirectory, Plugins, EngineModuleFiles, new List(), ModuleFileToPluginInfo, EngineAssemblyFileName, bContainsEngineModules: true, bUseBackwardsCompatibleDefaults: false, bReadOnly: bReadOnly, bSkipCompile: bSkipCompile, Parent: Parent); // Find all the rules files List ProgramModuleFiles; using(Timeline.ScopeEvent("Finding program modules")) { ProgramModuleFiles = new List(FindAllRulesFiles(ProgramsDirectory, RulesFileType.Module)); } List ProgramTargetFiles; using(Timeline.ScopeEvent("Finding program targets")) { ProgramTargetFiles = new List(FindAllRulesFiles(SourceDirectory, RulesFileType.Target)); } // Create a path to the assembly that we'll either load or compile FileReference ProgramAssemblyFileName = FileReference.Combine(RootDirectory, "Intermediate", "Build", "BuildRules", AssemblyPrefix + "ProgramRules" + FrameworkAssemblyExtension); RulesAssembly ProgramAssembly = new RulesAssembly(RootDirectory, new List().AsReadOnly(), ProgramModuleFiles, ProgramTargetFiles, new Dictionary(), ProgramAssemblyFileName, bContainsEngineModules: false, bUseBackwardsCompatibleDefaults: false, bReadOnly: bReadOnly, bSkipCompile: bSkipCompile, Parent: EngineAssembly); // Return the combined assembly return ProgramAssembly; } /// /// Creates a rules assembly with the given parameters. /// /// The project file to create rules for. Null for the engine. /// Whether to use a precompiled engine /// Whether to skip compilation for this assembly /// New rules assembly public static RulesAssembly CreateProjectRulesAssembly(FileReference ProjectFileName, bool bUsePrecompiled, bool bSkipCompile) { // Check if there's an existing assembly for this project RulesAssembly ProjectRulesAssembly; if (!LoadedAssemblyMap.TryGetValue(ProjectFileName, out ProjectRulesAssembly)) { ProjectDescriptor Project = ProjectDescriptor.FromFile(ProjectFileName); // Create the parent assembly RulesAssembly Parent; if (Project.IsEnterpriseProject) { Parent = CreateEnterpriseRulesAssembly(bUsePrecompiled, bSkipCompile); } else { Parent = CreateEngineRulesAssembly(bUsePrecompiled, bSkipCompile); } // Find all the rules under the project source directory DirectoryReference ProjectDirectory = ProjectFileName.Directory; DirectoryReference ProjectSourceDirectory = DirectoryReference.Combine(ProjectDirectory, "Source"); List ModuleFiles = new List(FindAllRulesFiles(ProjectSourceDirectory, RulesFileType.Module)); List TargetFiles = new List(FindAllRulesFiles(ProjectSourceDirectory, RulesFileType.Target)); // Find all the project plugins List ProjectPlugins = new List(Plugins.ReadProjectPlugins(ProjectFileName.Directory)); // Add the project's additional plugin directories plugins too if(Project.AdditionalPluginDirectories != null) { foreach(string AdditionalPluginDirectory in Project.AdditionalPluginDirectories) { ProjectPlugins.AddRange(Plugins.ReadAdditionalPlugins(DirectoryReference.Combine(ProjectFileName.Directory, AdditionalPluginDirectory))); } } // Find all the plugin module rules Dictionary ModuleFileToPluginInfo = new Dictionary(); FindModuleRulesForPlugins(ProjectPlugins, ModuleFiles, ModuleFileToPluginInfo); // Add the games project's intermediate source folder DirectoryReference ProjectIntermediateSourceDirectory = DirectoryReference.Combine(ProjectDirectory, "Intermediate", "Source"); if (DirectoryReference.Exists(ProjectIntermediateSourceDirectory)) { ModuleFiles.AddRange(FindAllRulesFiles(ProjectIntermediateSourceDirectory, RulesFileType.Module)); TargetFiles.AddRange(FindAllRulesFiles(ProjectIntermediateSourceDirectory, RulesFileType.Target)); } // Compile the assembly. If there are no module or target files, just use the parent assembly. FileReference AssemblyFileName = FileReference.Combine(ProjectDirectory, "Intermediate", "Build", "BuildRules", ProjectFileName.GetFileNameWithoutExtension() + "ModuleRules" + FrameworkAssemblyExtension); if(ModuleFiles.Count == 0 && TargetFiles.Count == 0) { ProjectRulesAssembly = Parent; } else { ProjectRulesAssembly = new RulesAssembly(ProjectDirectory, ProjectPlugins, ModuleFiles, TargetFiles, ModuleFileToPluginInfo, AssemblyFileName, bContainsEngineModules: false, bUseBackwardsCompatibleDefaults: true, bReadOnly: UnrealBuildTool.IsProjectInstalled(), bSkipCompile: bSkipCompile, Parent: Parent); } LoadedAssemblyMap.Add(ProjectFileName, ProjectRulesAssembly); } return ProjectRulesAssembly; } /// /// Creates a rules assembly with the given parameters. /// /// The plugin file to create rules for /// Whether to skip compilation for this assembly /// The parent rules assembly /// Whether the plugin contains engine modules. Used to initialize the default value for ModuleRules.bTreatAsEngineModule. /// The new rules assembly public static RulesAssembly CreatePluginRulesAssembly(FileReference PluginFileName, bool bSkipCompile, RulesAssembly Parent, bool bContainsEngineModules) { // Check if there's an existing assembly for this project RulesAssembly PluginRulesAssembly; if (!LoadedAssemblyMap.TryGetValue(PluginFileName, out PluginRulesAssembly)) { // Find all the rules source files List ModuleFiles = new List(); List TargetFiles = new List(); // Create a list of plugins for this assembly. If it already exists in the parent assembly, just create an empty assembly. List ForeignPlugins = new List(); if (Parent == null || !Parent.EnumeratePlugins().Any(x => x.File == PluginFileName)) { ForeignPlugins.Add(new PluginInfo(PluginFileName, PluginType.External)); } // Find all the modules Dictionary ModuleFileToPluginInfo = new Dictionary(); FindModuleRulesForPlugins(ForeignPlugins, ModuleFiles, ModuleFileToPluginInfo); // Compile the assembly FileReference AssemblyFileName = FileReference.Combine(PluginFileName.Directory, "Intermediate", "Build", "BuildRules", Path.GetFileNameWithoutExtension(PluginFileName.FullName) + "ModuleRules" + FrameworkAssemblyExtension); PluginRulesAssembly = new RulesAssembly(PluginFileName.Directory, ForeignPlugins, ModuleFiles, TargetFiles, ModuleFileToPluginInfo, AssemblyFileName, bContainsEngineModules, bUseBackwardsCompatibleDefaults: false, bReadOnly: false, bSkipCompile: bSkipCompile, Parent: Parent); LoadedAssemblyMap.Add(PluginFileName, PluginRulesAssembly); } return PluginRulesAssembly; } /// /// Compile a rules assembly for the current target /// /// The project file being compiled /// The target being built /// Whether to skip compiling any rules assemblies /// Whether to use a precompiled engine/enterprise build /// Foreign plugin to be compiled /// The compiled rules assembly public static RulesAssembly CreateTargetRulesAssembly(FileReference ProjectFile, string TargetName, bool bSkipRulesCompile, bool bUsePrecompiled, FileReference ForeignPlugin) { RulesAssembly RulesAssembly; if (ProjectFile != null) { RulesAssembly = CreateProjectRulesAssembly(ProjectFile, bUsePrecompiled, bSkipRulesCompile); } else { RulesAssembly = CreateEngineRulesAssembly(bUsePrecompiled, bSkipRulesCompile); if (RulesAssembly.GetTargetFileName(TargetName) == null && DirectoryReference.Exists(UnrealBuildTool.EnterpriseDirectory)) { // Target isn't part of the engine assembly, try the enterprise assembly RulesAssembly = CreateEnterpriseRulesAssembly(bUsePrecompiled, bSkipRulesCompile); } } if (ForeignPlugin != null) { RulesAssembly = CreatePluginRulesAssembly(ForeignPlugin, bSkipRulesCompile, RulesAssembly, true); } return RulesAssembly; } /// /// Finds all the module rules for plugins under the given directory. /// /// The directory to search /// List of module files to be populated /// Dictionary which is filled with mappings from the module file to its corresponding plugin file private static void FindModuleRulesForPlugins(IReadOnlyList Plugins, List ModuleFiles, Dictionary ModuleFileToPluginInfo) { PrefetchRulesFiles(Plugins.Select(x => DirectoryReference.Combine(x.Directory, "Source"))); foreach (PluginInfo Plugin in Plugins) { IReadOnlyList PluginModuleFiles = FindAllRulesFiles(DirectoryReference.Combine(Plugin.Directory, "Source"), RulesFileType.Module); foreach (FileReference ModuleFile in PluginModuleFiles) { ModuleFiles.Add(ModuleFile); ModuleFileToPluginInfo[ModuleFile] = Plugin; } } } /// /// Gets the filename that declares the given type. /// /// The type to search for. /// The filename that declared the given type, or null public static string GetFileNameFromType(Type ExistingType) { FileReference FileName; if (EngineRulesAssembly != null && EngineRulesAssembly.TryGetFileNameFromType(ExistingType, out FileName)) { return FileName.FullName; } else if (EnterpriseRulesAssembly != null && EnterpriseRulesAssembly.TryGetFileNameFromType(ExistingType, out FileName)) { return FileName.FullName; } foreach (RulesAssembly RulesAssembly in LoadedAssemblyMap.Values) { if (RulesAssembly.TryGetFileNameFromType(ExistingType, out FileName)) { return FileName.FullName; } } return null; } } }