// Copyright 1998-2017 Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.Serialization.Formatters.Binary; namespace UnrealBuildTool { /// /// Stores an ordered list of header files /// class FlatCPPIncludeDependencyInfo { /// The PCH header this file is dependent on. public FileReference PCHName; /// List of files this file includes, excluding headers that were included from a PCH. public List Includes; /// Transient cached list of FileItems for all of the includes for a specific files. This just saves a bunch of string /// hash lookups as we locate FileItems for files that we've already requested dependencies for public List IncludeFileItems; } /// /// For a given target, caches all of the C++ source files and the files they are including /// class FlatCPPIncludeDependencyCache { /// /// The version number for binary serialization /// const int FileVersion = 2; /// /// The file signature for binary serialization /// const int FileSignature = ('F' << 24) | ('C' << 16) | FileVersion; /// File name of this cache, should be unique for every includes context (e.g. target) private FileReference BackingFile; /// True if the cache needs to be saved private bool bIsDirty = false; /// Dependency lists, keyed (case-insensitively) on file's absolute path. private Dictionary DependencyMap; /// /// Constructs a fresh cache, storing the file name that it should be saved as later on /// /// File name for this cache, usually unique per context (e.g. target) public FlatCPPIncludeDependencyCache(FileReference BackingFile) { this.BackingFile = BackingFile; DependencyMap = new Dictionary(); } /// /// Loads the cache from disk /// /// The file to read from /// The loaded cache /// True if successful public static bool TryRead(FileReference BackingFile, out FlatCPPIncludeDependencyCache Cache) { if(!FileReference.Exists(BackingFile)) { Cache = null; return false; } if (UnrealBuildTool.bPrintPerformanceInfo) { Log.TraceInformation("Loading existing FlatCPPIncludeDependencyCache: " + BackingFile.FullName); } DateTime TimerStartTime = DateTime.UtcNow; try { // If the .buildmutex file for the cache is present, it means that something went wrong between loading // and saving the cache last time (most likely the UBT process being terminated), so we don't want to load // it. string CacheBuildMutexPath = BackingFile.FullName + ".buildmutex"; if (File.Exists(CacheBuildMutexPath)) { Cache = null; return false; } File.Create(CacheBuildMutexPath).Close(); using (BinaryReader Reader = new BinaryReader(new FileStream(BackingFile.FullName, FileMode.Open, FileAccess.Read))) { // @todo ubtmake: We can store the cache in a cheaper/smaller way using hash file names and indices into included headers, but it might actually slow down load times // @todo ubtmake: If we can index PCHs here, we can avoid storing all of the PCH's included headers (PCH's action should have been invalidated, so we shouldn't even have to report the PCH's includes as our indirect includes) if (Reader.ReadInt32() != FileSignature) { Cache = null; return false; } Cache = Deserialize(Reader); if (UnrealBuildTool.bPrintPerformanceInfo) { TimeSpan TimerDuration = DateTime.UtcNow - TimerStartTime; Log.TraceInformation("Loading FlatCPPIncludeDependencyCache took " + TimerDuration.TotalSeconds + "s"); } return true; } } catch (Exception Ex) { Console.Error.WriteLine("Failed to read FlatCPPIncludeDependencyCache: {0}", Ex.Message); FileReference.Delete(BackingFile); Cache = null; return false; } } /// /// Saves out the cache /// public void Save() { // Only save if we've made changes to it since load. if (bIsDirty) { DateTime TimerStartTime = DateTime.UtcNow; // Serialize the cache to disk. try { DirectoryReference.CreateDirectory(BackingFile.Directory); using (BinaryWriter Writer = new BinaryWriter(new FileStream(BackingFile.FullName, FileMode.Create, FileAccess.Write))) { Writer.Write(FileSignature); Serialize(Writer); } } catch (Exception Ex) { Console.Error.WriteLine("Failed to write FlatCPPIncludeDependencyCache: {0}", Ex.Message); } if (UnrealBuildTool.bPrintPerformanceInfo) { TimeSpan TimerDuration = DateTime.UtcNow - TimerStartTime; Log.TraceInformation("Saving FlatCPPIncludeDependencyCache took " + TimerDuration.TotalSeconds + "s"); } } else { if (UnrealBuildTool.bPrintPerformanceInfo) { Log.TraceInformation("FlatCPPIncludeDependencyCache did not need to be saved (bIsDirty=false)"); } } if (File.Exists(BackingFile.FullName + ".buildmutex")) { try { File.Delete(BackingFile.FullName + ".buildmutex"); } catch { // We don't care if we couldn't delete this file, as maybe it couldn't have been created in the first place. } } } /// /// Serializes the dependency cache to a binary writer /// /// Writer to output to void Serialize(BinaryWriter Writer) { Dictionary FileToUniqueId = new Dictionary(); Writer.Write(BackingFile); // Write out all the dependency info objects Writer.Write(DependencyMap.Count); foreach (KeyValuePair Pair in DependencyMap) { Writer.Write(Pair.Key); FlatCPPIncludeDependencyInfo Info = Pair.Value; Writer.Write(Info.PCHName, FileToUniqueId); Writer.Write(Info.Includes.Count); foreach (FileReference Include in Info.Includes) { Writer.Write(Include, FileToUniqueId); } } } /// /// Deserialize the dependency cache from a binary reader /// /// Reader for the cache data /// New dependency cache object static FlatCPPIncludeDependencyCache Deserialize(BinaryReader Reader) { FlatCPPIncludeDependencyCache Cache = new FlatCPPIncludeDependencyCache(Reader.ReadFileReference()); // Create the dependency map int NumDependencyMapEntries = Reader.ReadInt32(); Cache.DependencyMap = new Dictionary(NumDependencyMapEntries); // Read all the dependency map entries List UniqueFiles = new List(NumDependencyMapEntries * 2); for (int Idx = 0; Idx < NumDependencyMapEntries; Idx++) { FileReference File = Reader.ReadFileReference(); FlatCPPIncludeDependencyInfo Info = new FlatCPPIncludeDependencyInfo(); // Read the PCH name Info.PCHName = Reader.ReadFileReference(UniqueFiles); // Read the includes int NumIncludes = Reader.ReadInt32(); Info.Includes = new List(NumIncludes); for (int IncludeIdx = 0; IncludeIdx < NumIncludes; IncludeIdx++) { Info.Includes.Add(Reader.ReadFileReference(UniqueFiles)); } // Add it to the map Cache.DependencyMap.Add(File, Info); } return Cache; } /// /// Sets the new dependencies for the specified file /// /// File to set /// The PCH for this file /// List of source dependencies public void SetDependenciesForFile(FileReference AbsoluteFilePath, FileReference PCHName, List Dependencies) { FlatCPPIncludeDependencyInfo DependencyInfo = new FlatCPPIncludeDependencyInfo(); DependencyInfo.PCHName = PCHName; // @todo ubtmake: Not actually used yet. The idea is to use this to reduce the number of indirect includes we need to store in the cache. DependencyInfo.Includes = Dependencies; DependencyInfo.IncludeFileItems = null; // @todo ubtmake: We could shrink this file by not storing absolute paths (project and engine relative paths only, except for system headers.) May affect load times. DependencyMap[AbsoluteFilePath] = DependencyInfo; bIsDirty = true; } /// /// Gets everything that this file includes from our cache (direct and indirect!) /// /// Path to the file /// The list of includes public List GetDependenciesForFile(FileReference AbsoluteFilePath) { FlatCPPIncludeDependencyInfo DependencyInfo; if (DependencyMap.TryGetValue(AbsoluteFilePath, out DependencyInfo)) { // Update our transient cache of FileItems for each of the included files if (DependencyInfo.IncludeFileItems == null) { DependencyInfo.IncludeFileItems = new List(DependencyInfo.Includes.Count); foreach (FileReference Dependency in DependencyInfo.Includes) { DependencyInfo.IncludeFileItems.Add(FileItem.GetItemByFileReference(Dependency)); } } return DependencyInfo.IncludeFileItems; } return null; } /// /// Gets the dependency cache path and filename for the specified target. /// /// Current build target /// Cache Path public static FileReference GetDependencyCachePathForTarget(UEBuildTarget Target) { DirectoryReference PlatformIntermediatePath; if (Target.ProjectFile != null) { PlatformIntermediatePath = DirectoryReference.Combine(Target.ProjectFile.Directory, Target.PlatformIntermediateFolder); } else { PlatformIntermediatePath = DirectoryReference.Combine(UnrealBuildTool.EngineDirectory, Target.PlatformIntermediateFolder); } return FileReference.Combine(PlatformIntermediatePath, Target.GetTargetName(), "FlatCPPIncludes.bin"); } } }