Files
UnrealEngineUWP/Engine/Source/Programs/UnrealBuildTool/ToolChain/UEToolChain.cs
henrik karlsson 5f74d114d5 [UBT]
Changed how single files are built. We don't want to invalidate makefile everytime we build single files since it destroys turnaround times. With this change a single file compile can take 2s (+ the actual compile time).

The new behavior injects a special action per module when creating the makefile. These actions can be used to on-the-fly create a proper compile action that follows the rules of the module that the specific file belongs to.. In a normal build these actions are ignored since the logic deciding which actions to build is backtraced from which binaries we want to create.

When a specific file compile is triggered, the logic deciding which files to build search up all these special actions and create a lookup based on which folders the special actions handle. It then try to find the special action that handles the specific file. The matching special action then creates a compile action that can handle that specific file and then queue up the action for execution. If no special action is found it falls back to try to use actions that have this specific file as input (ispc files for example)

Details:
* Removed lots of custom code for "specific files" handling
* Changed so pch (both private and shared) always use definition file. Added #pragma once and change so pch wrapper file include definition file. This made the adaptive path and specific file easy to implement (just disable pch in compile environment and it will just work)
* Added SingleFileAction for both VCToolChain and ClangToolChain. It now works to compile specific headers and cpp files. (It creates wrapper files on the fly to be able to compile all header files (compiling headers directly blow up if there are circular includes)
* Fixed so GenerateClangDatabase mode works with new changes
* Moved the logic that makes sure all (directly) depending cpp files are recompiled when .h are included in the singlefile option

#preflight 63dcc46f78716a01e8069649
#rb joe.kirchoff

[CL 24094027 by henrik karlsson in ue5-main branch]
2023-02-09 04:20:43 -05:00

288 lines
11 KiB
C#

// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Linq;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using EpicGames.Core;
using Microsoft.Extensions.Logging;
using UnrealBuildBase;
namespace UnrealBuildTool
{
abstract class UEToolChain
{
protected readonly ILogger Logger;
// Return the extension for response files
public static string ResponseExt => ".rsp";
public UEToolChain(ILogger InLogger)
{
Logger = InLogger;
}
public virtual void SetEnvironmentVariables()
{
}
public virtual void GetVersionInfo(List<string> Lines)
{
}
public virtual void GetExternalDependencies(HashSet<FileItem> ExternalDependencies)
{
}
public static DirectoryReference GetModuleInterfaceDir(DirectoryReference OutputDir)
{
return DirectoryReference.Combine(OutputDir, "Ifc");
}
// Return the path to the cpp compiler that will be used by this toolchain.
public virtual FileReference? GetCppCompilerPath()
{
return null;
}
protected abstract CPPOutput CompileCPPFiles(CppCompileEnvironment CompileEnvironment, List<FileItem> InputFiles, DirectoryReference OutputDir, string ModuleName, IActionGraphBuilder Graph);
public CPPOutput CompileAllCPPFiles(CppCompileEnvironment CompileEnvironment, List<FileItem> InputFiles, DirectoryReference OutputDir, string ModuleName, IActionGraphBuilder Graph)
{
CPPOutput Result;
UnrealArchitectureConfig ArchConfig = UnrealArchitectureConfig.ForPlatform(CompileEnvironment.Platform);
// compile architectures separately if needed
if (ArchConfig.Mode == UnrealArchitectureMode.SingleTargetCompileSeparately || ArchConfig.Mode == UnrealArchitectureMode.SingleTargetLinkSeparately)
{
Result = new CPPOutput();
foreach (UnrealArch Arch in CompileEnvironment.Architectures.Architectures)
{
// determine the output location of intermediates (so, if OutputDir had the arch name in it, like Intermediate/x86+arm64, we would replace it with either emptry string
// or a single arch name depending on if the platform uses architecture directories for the architecture)
// @todo Add ArchitectureConfig.RequiresArchitectureFilenames but for directory -- or can we just use GetFolderNameForArch?!?!?
// string ArchReplacement = (Arch == ArchitectureWithoutMarkup()) ? "" : ArchConfig.GetFolderNameForArchitecture(Arch);
string PlatformArchitecturesString = ArchConfig.GetFolderNameForArchitectures(CompileEnvironment.Architectures);
DirectoryReference ArchOutputDir = new(OutputDir.FullName.Replace(PlatformArchitecturesString, ArchConfig.GetFolderNameForArchitecture(Arch)));
CppCompileEnvironment ArchEnvironment = new(CompileEnvironment, Arch);
CPPOutput ArchResult = CompileCPPFiles(ArchEnvironment, InputFiles, ArchOutputDir, ModuleName, Graph);
Result.Merge(ArchResult, Arch);
}
}
else
{
Result = CompileCPPFiles(CompileEnvironment, InputFiles, OutputDir, ModuleName, Graph);
}
return Result;
}
public virtual CPPOutput CompileRCFiles(CppCompileEnvironment Environment, List<FileItem> InputFiles, DirectoryReference OutputDir, IActionGraphBuilder Graph)
{
CPPOutput Result = new CPPOutput();
return Result;
}
public virtual CPPOutput CompileISPCFiles(CppCompileEnvironment Environment, List<FileItem> InputFiles, DirectoryReference OutputDir, IActionGraphBuilder Graph)
{
CPPOutput Result = new CPPOutput();
return Result;
}
public virtual CPPOutput GenerateISPCHeaders(CppCompileEnvironment Environment, List<FileItem> InputFiles, DirectoryReference OutputDir, IActionGraphBuilder Graph)
{
CPPOutput Result = new CPPOutput();
return Result;
}
public virtual void GenerateTypeLibraryHeader(CppCompileEnvironment CompileEnvironment, ModuleRules.TypeLibrary TypeLibrary, FileReference OutputFile, IActionGraphBuilder Graph)
{
throw new NotSupportedException("This platform does not support type libraries.");
}
/// <summary>
/// Allows a toolchain to decide to create an import library if needed for this Environment
/// </summary>
/// <param name="LinkEnvironment"></param>
/// <param name="Graph"></param>
/// <returns></returns>
public virtual FileItem[] LinkImportLibrary(LinkEnvironment LinkEnvironment, IActionGraphBuilder Graph)
{
// by default doing nothing
return new FileItem[] { };
}
public abstract FileItem? LinkFiles(LinkEnvironment LinkEnvironment, bool bBuildImportLibraryOnly, IActionGraphBuilder Graph);
public virtual FileItem[] LinkAllFiles(LinkEnvironment LinkEnvironment, bool bBuildImportLibraryOnly, IActionGraphBuilder Graph)
{
List<FileItem> Result = new();
// compile architectures separately if needed
UnrealArchitectureConfig ArchConfig = UnrealArchitectureConfig.ForPlatform(LinkEnvironment.Platform);
if (ArchConfig.Mode == UnrealArchitectureMode.SingleTargetLinkSeparately)
{
foreach (UnrealArch Arch in LinkEnvironment.Architectures.Architectures)
{
LinkEnvironment ArchEnvironment = new LinkEnvironment(LinkEnvironment, Arch);
// determine the output location of intermediates (so, if OutputDir had the arch name in it, like Intermediate/x86+arm64, we would replace it with either emptry string
// or a single arch name
//string ArchReplacement = Arch == ArchitectureWithoutMarkup() ? "" : ArchConfig.GetFolderNameForArchitecture(Arch);
string PlatformArchitecturesString = ArchConfig.GetFolderNameForArchitectures(LinkEnvironment.Architectures);
ArchEnvironment.OutputFilePaths = LinkEnvironment.OutputFilePaths.Select(x => new FileReference(x.FullName.Replace(PlatformArchitecturesString, ArchConfig.GetFolderNameForArchitecture(Arch)))).ToList();
FileItem? LinkFile = LinkFiles(ArchEnvironment, bBuildImportLibraryOnly, Graph);
if (LinkFile != null)
{
Result.Add(LinkFile);
}
}
}
else
{
FileItem? LinkFile = LinkFiles(LinkEnvironment, bBuildImportLibraryOnly, Graph);
if (LinkFile != null)
{
Result.Add(LinkFile);
}
}
return Result.ToArray();
}
public virtual CppCompileEnvironment CreateSharedResponseFile(CppCompileEnvironment CompileEnvironment, FileReference OutResponseFile, IActionGraphBuilder Graph)
{
return CompileEnvironment;
}
public virtual void CreateSpecificFileAction(CppCompileEnvironment CompileEnvironment, DirectoryReference SourceDir, DirectoryReference OutputDir, IActionGraphBuilder Graph)
{
}
/// <summary>
/// Get the name of the response file for the current compile environment and output file
/// </summary>
/// <param name="CompileEnvironment"></param>
/// <param name="OutputFile"></param>
/// <returns></returns>
public static FileReference GetResponseFileName(CppCompileEnvironment CompileEnvironment, FileItem OutputFile)
{
// Construct a relative path for the intermediate response file
return OutputFile.Location.ChangeExtension(OutputFile.Location.GetExtension() + ResponseExt);
}
/// <summary>
/// Get the name of the response file for the current linker environment and output file
/// </summary>
/// <param name="LinkEnvironment"></param>
/// <param name="OutputFile"></param>
/// <returns></returns>
public static FileReference GetResponseFileName(LinkEnvironment LinkEnvironment, FileItem OutputFile)
{
// Construct a relative path for the intermediate response file
return FileReference.Combine(LinkEnvironment.IntermediateDirectory!, OutputFile.Location.GetFileName() + ResponseExt);
}
public virtual ICollection<FileItem> PostBuild(ReadOnlyTargetRules Target, FileItem Executable, LinkEnvironment ExecutableLinkEnvironment, IActionGraphBuilder Graph)
{
return new List<FileItem>();
}
public virtual ICollection<FileItem> PostBuild(ReadOnlyTargetRules Target, FileItem[] Executables, LinkEnvironment ExecutableLinkEnvironment, IActionGraphBuilder Graph)
{
// by default, run PostBuild for exe Exe and merge results
return Executables.SelectMany(x => PostBuild(Target, x, ExecutableLinkEnvironment, Graph)).ToList();
}
public virtual void SetUpGlobalEnvironment(ReadOnlyTargetRules Target)
{
}
public virtual void ModifyBuildProducts(ReadOnlyTargetRules Target, UEBuildBinary Binary, List<string> Libraries, List<UEBuildBundleResource> BundleResources, Dictionary<FileReference, BuildProductType> BuildProducts)
{
}
public virtual void ModifyTargetReceipt(TargetReceipt Receipt)
{
}
public virtual void FinalizeOutput(ReadOnlyTargetRules Target, TargetMakefileBuilder MakefileBuilder)
{
}
public virtual void PrepareRuntimeDependencies(List<RuntimeDependency> RuntimeDependencies, Dictionary<FileReference, FileReference> TargetFileToSourceFile, DirectoryReference ExeDir)
{
}
/// <summary>
/// Adds a build product and its associated debug file to a receipt.
/// </summary>
/// <param name="OutputFile">Build product to add</param>
/// <param name="OutputType">The type of build product</param>
public virtual bool ShouldAddDebugFileToReceipt(FileReference OutputFile, BuildProductType OutputType)
{
return true;
}
public virtual FileReference GetDebugFile(FileReference OutputFile, string DebugExtension)
{
// by default, just change the extension to the debug extension
return OutputFile.ChangeExtension(DebugExtension);
}
public virtual void SetupBundleDependencies(ReadOnlyTargetRules Target, List<UEBuildBinary> Binaries, string GameName)
{
}
public virtual string GetSDKVersion()
{
return "Not Applicable";
}
/// <summary>
/// Runs the provided tool and argument. Returns the output, using a rexex capture if one is provided
/// </summary>
/// <param name="Command">Full path to the tool to run</param>
/// <param name="ToolArg">Argument that will be passed to the tool</param>
/// <param name="Expression">null, or a Regular expression to capture in the output</param>
/// <returns></returns>
protected string? RunToolAndCaptureOutput(FileReference Command, string ToolArg, string? Expression = null)
{
string ProcessOutput = Utils.RunLocalProcessAndReturnStdOut(Command.FullName, ToolArg, Logger);
if (string.IsNullOrEmpty(Expression))
{
return ProcessOutput;
}
Match M = Regex.Match(ProcessOutput, Expression);
return M.Success ? M.Groups[1].ToString() : null;
}
/// <summary>
/// Runs the provided tool and argument and parses the output to retrieve the version
/// </summary>
/// <param name="Command">Full path to the tool to run</param>
/// <param name="VersionArg">Argument that will result in the version string being shown (it's ok this is a byproduct of a command that returns an error)</param>
/// <param name="VersionExpression">Regular expression to capture the version. By default we look for four integers separated by periods, with the last two optional</param>
/// <returns></returns>
public Version RunToolAndCaptureVersion(FileReference Command, string VersionArg, string VersionExpression = @"(\d+\.\d+(\.\d+)?(\.\d+)?)")
{
string? ProcessOutput = RunToolAndCaptureOutput(Command, VersionArg, VersionExpression);
Version? ToolVersion;
if (Version.TryParse(ProcessOutput, out ToolVersion))
{
return ToolVersion;
}
Logger.LogWarning("Unable to retrieve version from {Command} {Arg}", Command, VersionArg);
return new Version(0, 0);
}
};
}