// Copyright 1998-2018 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.Text;
using System.Diagnostics;
using System.IO;
using Tools.DotNETCommon;
using System.Text.RegularExpressions;
namespace UnrealBuildTool
{
/** Architecture as stored in the ini. */
enum LinuxArchitecture
{
/** x86_64, most commonly used architecture.*/
X86_64UnknownLinuxGnu,
/** A.k.a. AArch32, ARM 32-bit with hardware floats */
ArmUnknownLinuxGnueabihf,
/** AArch64, ARM 64-bit */
AArch64UnknownLinuxGnueabi,
/** i686, Intel 32-bit */
I686UnknownLinuxGnu
}
class LinuxPlatform : UEBuildPlatform
{
///
/// Linux architecture (compiler target triplet)
///
// FIXME: for now switching between architectures is hard-coded
public const string DefaultArchitecture = "x86_64-unknown-linux-gnu";
//public const string DefaultArchitecture = "arm-unknown-linux-gnueabihf";
//public const string DefaultArchitecture = "aarch64-unknown-linux-gnueabi";
///
/// SDK in use by the platform
///
protected LinuxPlatformSDK SDK;
///
/// Constructor
///
public LinuxPlatform(LinuxPlatformSDK InSDK)
: this(UnrealTargetPlatform.Linux, CppPlatform.Linux, InSDK)
{
SDK = InSDK;
}
public LinuxPlatform(UnrealTargetPlatform UnrealTarget, CppPlatform InCppPlatform, LinuxPlatformSDK InSDK)
: base(UnrealTarget, InCppPlatform)
{
SDK = InSDK;
}
///
/// Whether the required external SDKs are installed for this platform. Could be either a manual install or an AutoSDK.
///
public override SDKStatus HasRequiredSDKsInstalled()
{
return SDK.HasRequiredSDKsInstalled();
}
///
/// Find the default architecture for the given project
///
public override string GetDefaultArchitecture(FileReference ProjectFile)
{
string ActiveArchitecture = DefaultArchitecture;
// read settings from the config
string EngineIniPath = ProjectFile != null ? ProjectFile.Directory.FullName : null;
if (String.IsNullOrEmpty(EngineIniPath))
{
// If the project file hasn't been specified, try to get the path from -remoteini command line param
EngineIniPath = UnrealBuildTool.GetRemoteIniPath();
}
DirectoryReference EngineIniDir = !String.IsNullOrEmpty(EngineIniPath) ? new DirectoryReference(EngineIniPath) : null;
ConfigHierarchy Ini = ConfigCache.ReadHierarchy(ConfigHierarchyType.Engine, EngineIniDir, UnrealTargetPlatform.Linux);
string LinuxArchitectureString;
if (Ini.GetString("/Script/LinuxTargetPlatform.LinuxTargetSettings", "TargetArchitecture", out LinuxArchitectureString))
{
LinuxArchitecture Architecture;
if (Enum.TryParse(LinuxArchitectureString, out Architecture))
{
switch (Architecture)
{
default:
System.Console.WriteLine("Architecture enum value {0} does not map to a valid triplet.", Architecture);
break;
case LinuxArchitecture.X86_64UnknownLinuxGnu:
ActiveArchitecture = "x86_64-unknown-linux-gnu";
break;
case LinuxArchitecture.ArmUnknownLinuxGnueabihf:
ActiveArchitecture = "arm-unknown-linux-gnueabihf";
break;
case LinuxArchitecture.AArch64UnknownLinuxGnueabi:
ActiveArchitecture = "aarch64-unknown-linux-gnueabi";
break;
case LinuxArchitecture.I686UnknownLinuxGnu:
ActiveArchitecture = "i686-unknown-linux-gnu";
break;
}
}
}
return ActiveArchitecture;
}
///
/// Get name for architecture-specific directories (can be shorter than architecture name itself)
///
public override string GetFolderNameForArchitecture(string Architecture)
{
// shorten the string (heuristically)
uint Sum = 0;
int Len = Architecture.Length;
for (int Index = 0; Index < Len; ++Index)
{
Sum += (uint)(Architecture[Index]);
Sum <<= 1; // allowed to overflow
}
return Sum.ToString("X");
}
public override void ResetTarget(TargetRules Target)
{
ValidateTarget(Target);
}
public override void ValidateTarget(TargetRules Target)
{
Target.bCompileSimplygon = false;
Target.bCompileSimplygonSSF = false;
// depends on arch, APEX cannot be as of November'16 compiled for AArch32/64
Target.bCompileAPEX = Target.Architecture.StartsWith("x86_64");
Target.bCompileNvCloth = Target.Architecture.StartsWith("x86_64");
// Disable Simplygon support if compiling against the NULL RHI.
if (Target.GlobalDefinitions.Contains("USE_NULL_RHI=1"))
{
Target.bCompileSimplygon = false;
Target.bCompileSimplygonSSF = false;
Target.bCompileCEF3 = false;
}
// check if OS update invalidated our build
Target.bCheckSystemHeadersForModification = (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux);
// At the moment ICU has not been compiled for AArch64 and i686. Also, localization isn't needed on servers by default, and ICU is pretty heavy
if (Target.Architecture.StartsWith("aarch64") || Target.Architecture.StartsWith("i686") || Target.Type == TargetType.Server)
{
Target.bCompileICU = false;
}
if (ProjectFileGenerator.bGenerateProjectFiles)
{
// When generating project files we need intellisense generator to include info from all modules,
// including editor-only third party libs
Target.bCompileLeanAndMeanUE = false;
}
}
///
/// Allows the platform to override whether the architecture name should be appended to the name of binaries.
///
/// True if the architecture name should be appended to the binary
public override bool RequiresArchitectureSuffix()
{
// Linux ignores architecture-specific names, although it might be worth it to prepend architecture
return false;
}
public override bool CanUseXGE()
{
// XGE crashes with very high probability when v8_clang-3.9.0-centos cross-toolchain is used on Windows. Please make sure this is resolved before re-enabling it.
// [RCL] 2018-05-02: disabling XGE even during a native build because the support is not ready and you can have mysterious build failures when ib_console is installed.
return false;//BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux;
}
public override bool CanUseParallelExecutor()
{
// No known problems with parallel executor, always use for build machines
return true;
}
///
/// Determines if the given name is a build product for a target.
///
/// The name to check
/// Target or application names that may appear at the start of the build product name (eg. "UE4Editor", "ShooterGameEditor")
/// Suffixes which may appear at the end of the build product name
/// True if the string matches the name of a build product, false otherwise
public override bool IsBuildProduct(string FileName, string[] NamePrefixes, string[] NameSuffixes)
{
if (FileName.StartsWith("lib"))
{
return IsBuildProductName(FileName, 3, FileName.Length - 3, NamePrefixes, NameSuffixes, ".a")
|| IsBuildProductName(FileName, 3, FileName.Length - 3, NamePrefixes, NameSuffixes, ".so");
}
else
{
return IsBuildProductName(FileName, NamePrefixes, NameSuffixes, "")
|| IsBuildProductName(FileName, NamePrefixes, NameSuffixes, ".so")
|| IsBuildProductName(FileName, NamePrefixes, NameSuffixes, ".a");
}
}
///
/// Get the extension to use for the given binary type
///
/// The binary type being built
/// string The binary extension (i.e. 'exe' or 'dll')
public override string GetBinaryExtension(UEBuildBinaryType InBinaryType)
{
switch (InBinaryType)
{
case UEBuildBinaryType.DynamicLinkLibrary:
return ".so";
case UEBuildBinaryType.Executable:
return "";
case UEBuildBinaryType.StaticLibrary:
return ".a";
}
return base.GetBinaryExtension(InBinaryType);
}
///
/// Get the extensions to use for debug info for the given binary type
///
/// Rules for the target being built
/// The binary type being built
/// string[] The debug info extensions (i.e. 'pdb')
public override string[] GetDebugInfoExtensions(ReadOnlyTargetRules InTarget, UEBuildBinaryType InBinaryType)
{
switch (InBinaryType)
{
case UEBuildBinaryType.DynamicLinkLibrary:
case UEBuildBinaryType.Executable:
return new string[] {".sym", ".debug"};
}
return new string [] {};
}
///
/// Modify the rules for a newly created module, where the target is a different host platform.
/// This is not required - but allows for hiding details of a particular platform.
///
/// The name of the module
/// The module rules
/// The target being build
public override void ModifyModuleRulesForOtherPlatform(string ModuleName, ModuleRules Rules, ReadOnlyTargetRules Target)
{
if ((Target.Platform == UnrealTargetPlatform.Win32) || (Target.Platform == UnrealTargetPlatform.Win64))
{
if (!Target.bBuildRequiresCookedData)
{
if (ModuleName == "Engine")
{
if (Target.bBuildDeveloperTools)
{
Rules.DynamicallyLoadedModuleNames.Add("LinuxTargetPlatform");
Rules.DynamicallyLoadedModuleNames.Add("LinuxNoEditorTargetPlatform");
Rules.DynamicallyLoadedModuleNames.Add("LinuxClientTargetPlatform");
Rules.DynamicallyLoadedModuleNames.Add("LinuxServerTargetPlatform");
}
}
}
// allow standalone tools to use targetplatform modules, without needing Engine
if (Target.bForceBuildTargetPlatforms && ModuleName == "TargetPlatform")
{
Rules.DynamicallyLoadedModuleNames.Add("LinuxTargetPlatform");
Rules.DynamicallyLoadedModuleNames.Add("LinuxNoEditorTargetPlatform");
Rules.DynamicallyLoadedModuleNames.Add("LinuxClientTargetPlatform");
Rules.DynamicallyLoadedModuleNames.Add("LinuxServerTargetPlatform");
}
}
}
///
/// Converts the passed in path from UBT host to compiler native format.
///
public override string ConvertPath(string OriginalPath)
{
return LinuxToolChain.ConvertPath(OriginalPath);
}
///
/// Modify the rules for a newly created module, in a target that's being built for this platform.
/// This is not required - but allows for hiding details of a particular platform.
///
/// The name of the module
/// The module rules
/// The target being build
public override void ModifyModuleRulesForActivePlatform(string ModuleName, ModuleRules Rules, ReadOnlyTargetRules Target)
{
bool bBuildShaderFormats = Target.bForceBuildShaderFormats;
if (!Target.bBuildRequiresCookedData)
{
if (ModuleName == "TargetPlatform")
{
bBuildShaderFormats = true;
}
}
// allow standalone tools to use target platform modules, without needing Engine
if (ModuleName == "TargetPlatform")
{
if (Target.bForceBuildTargetPlatforms)
{
Rules.DynamicallyLoadedModuleNames.Add("LinuxTargetPlatform");
Rules.DynamicallyLoadedModuleNames.Add("LinuxNoEditorTargetPlatform");
Rules.DynamicallyLoadedModuleNames.Add("LinuxClientTargetPlatform");
Rules.DynamicallyLoadedModuleNames.Add("LinuxServerTargetPlatform");
Rules.DynamicallyLoadedModuleNames.Add("AllDesktopTargetPlatform");
}
if (bBuildShaderFormats)
{
Rules.DynamicallyLoadedModuleNames.Add("ShaderFormatOpenGL");
Rules.DynamicallyLoadedModuleNames.Add("VulkanShaderFormat");
}
}
}
public virtual void SetUpSpecificEnvironment(CppCompileEnvironment CompileEnvironment, LinkEnvironment LinkEnvironment)
{
CompileEnvironment.Definitions.Add("PLATFORM_LINUX=1");
CompileEnvironment.Definitions.Add("PLATFORM_UNIX=1");
CompileEnvironment.Definitions.Add("LINUX=1"); // For libOGG
// this define does not set jemalloc as default, just indicates its support
CompileEnvironment.Definitions.Add("PLATFORM_SUPPORTS_JEMALLOC=1");
}
///
/// Setup the target environment for building
///
/// Settings for the target being compiled
/// The compile environment for this target
/// The link environment for this target
public override void SetUpEnvironment(ReadOnlyTargetRules Target, CppCompileEnvironment CompileEnvironment, LinkEnvironment LinkEnvironment)
{
CompileEnvironment.Definitions.Add("WITH_DATABASE_SUPPORT=0"); //@todo linux: valid?
// During the native builds, check the system includes as well (check toolchain when cross-compiling?)
string BaseLinuxPath = SDK.GetBaseLinuxPathForArchitecture(Target.Architecture);
if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux && String.IsNullOrEmpty(BaseLinuxPath))
{
CompileEnvironment.IncludePaths.SystemIncludePaths.Add(new DirectoryReference("/usr/include"));
}
if (Target.Architecture.StartsWith("arm")) // AArch64 doesn't strictly need that - aligned access improves perf, but this will be likely offset by memcpys we're doing to guarantee it.
{
CompileEnvironment.Definitions.Add("REQUIRES_ALIGNED_INT_ACCESS");
}
if (CompileEnvironment.bAllowLTCG != LinkEnvironment.bAllowLTCG)
{
Log.TraceWarning("Inconsistency between LTCG settings in Compile and Link environments: link one takes priority");
CompileEnvironment.bAllowLTCG = LinkEnvironment.bAllowLTCG;
}
// disable to LTO for modular builds
if (CompileEnvironment.bAllowLTCG && Target.LinkType != TargetLinkType.Monolithic)
{
Log.TraceWarning("LTO (LTCG) for modular builds is not supported, disabling it");
CompileEnvironment.bAllowLTCG = false;
LinkEnvironment.bAllowLTCG = false;
}
// link with Linux libraries.
LinkEnvironment.AdditionalLibraries.Add("pthread");
// let this class or a sub class do settings specific to that class
SetUpSpecificEnvironment(CompileEnvironment, LinkEnvironment);
}
///
/// Whether this platform should create debug information or not
///
/// The target being built
/// bool true if debug info should be generated, false if not
public override bool ShouldCreateDebugInfo(ReadOnlyTargetRules Target)
{
switch (Target.Configuration)
{
case UnrealTargetConfiguration.Development:
case UnrealTargetConfiguration.Shipping:
case UnrealTargetConfiguration.Test:
case UnrealTargetConfiguration.Debug:
default:
return true;
};
}
///
/// Creates a toolchain instance for the given platform.
///
/// The platform to create a toolchain for
/// The target being built
/// New toolchain instance.
public override UEToolChain CreateToolChain(CppPlatform CppPlatform, ReadOnlyTargetRules Target)
{
return new LinuxToolChain(Target.Architecture, SDK);
}
///
/// Deploys the given target
///
/// Information about the target being deployed
public override void Deploy(UEBuildDeployTarget Target)
{
}
}
class LinuxPlatformSDK : UEBuildPlatformSDK
{
///
/// This is the SDK version we support
///
static string ExpectedSDKVersion = "v11_clang-5.0.0-centos7"; // now unified for all the architectures
///
/// Platform name (embeds architecture for now)
///
static private string TargetPlatformName = "Linux_x64";
///
/// Whether to compile with the verbose flag
///
public bool bVerboseCompiler = false;
///
/// Whether to link with the verbose flag
///
public bool bVerboseLinker = false;
///
/// Whether platform supports switching SDKs during runtime
///
/// true if supports
protected override bool PlatformSupportsAutoSDKs()
{
return true;
}
///
/// Returns platform-specific name used in SDK repository
///
/// path to SDK Repository
public override string GetSDKTargetPlatformName()
{
return TargetPlatformName;
}
///
/// Returns SDK string as required by the platform
///
/// Valid SDK string
protected override string GetRequiredSDKString()
{
return ExpectedSDKVersion;
}
protected override String GetRequiredScriptVersionString()
{
return "3.0";
}
protected override bool PreferAutoSDK()
{
// having LINUX_ROOT set (for legacy reasons or for convenience of cross-compiling certain third party libs) should not make UBT skip AutoSDKs
return true;
}
public string HaveLinuxDependenciesFile()
{
// This file must have no extension so that GitDeps considers it a binary dependency - it will only be pulled by the Setup script if Linux is enabled.
return "HaveLinuxDependencies";
}
public string SDKVersionFileName()
{
return "ToolchainVersion.txt";
}
protected static int GetLinuxToolchainVersionFromString(string SDKVersion)
{
// Example: v11_clang-5.0.0-centos7
string FullVersionPattern = @"^v[0-9]+_.*$";
Regex Regex = new Regex(FullVersionPattern);
if (Regex.IsMatch(SDKVersion))
{
string VersionPattern = @"[0-9]+";
Regex = new Regex(VersionPattern);
Match Match = Regex.Match(SDKVersion);
if (Match.Success)
{
int Version;
bool bParsed = Int32.TryParse(Match.Value, out Version);
if (bParsed)
{
return Version;
}
}
}
return -1;
}
public bool CheckSDKCompatible(string VersionString, out string ErrorMessage)
{
int Version = GetLinuxToolchainVersionFromString(VersionString);
int ExpectedVersion = GetLinuxToolchainVersionFromString(ExpectedSDKVersion);
if (Version >= 0 && ExpectedVersion >= 0 && Version != ExpectedVersion)
{
if (Version < ExpectedVersion)
{
ErrorMessage = "Toolchain found \"" + VersionString + "\" is older then the required version \"" + ExpectedSDKVersion + "\"";
return false;
}
else
{
Log.TraceWarning("Toolchain \"{0}\" is newer than the expected version \"{1}\", you may run into compilation errors", VersionString, ExpectedSDKVersion);
}
}
else if (VersionString != ExpectedSDKVersion)
{
ErrorMessage = "Failed to find a supported toolchain, found \"" + VersionString + "\", expected \"" + ExpectedSDKVersion + "\"";
return false;
}
ErrorMessage = "";
return true;
}
///
/// Returns the in-tree root for the Linux Toolchain for this host platform.
///
private static DirectoryReference GetInTreeSDKRoot()
{
return DirectoryReference.Combine(UnrealBuildTool.RootDirectory, "Engine/Extras/ThirdPartyNotUE/SDKs", "Host" + BuildHostPlatform.Current.Platform, TargetPlatformName);
}
///
/// Whether a host can use its system sdk for this platform
///
public virtual bool CanUseSystemCompiler()
{
return (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux);
}
///
/// Returns the root SDK path for all architectures
/// WARNING: Do not cache this value - it may be changed after sourcing OutputEnvVars.txt
///
/// Valid SDK string
public virtual string GetSDKLocation()
{
// if new multi-arch toolchain is used, prefer it
string MultiArchRoot = Environment.GetEnvironmentVariable("LINUX_MULTIARCH_ROOT");
if (String.IsNullOrEmpty(MultiArchRoot))
{
// check if in-tree SDK is available
DirectoryReference InTreeSDKVersionRoot = GetInTreeSDKRoot();
if (InTreeSDKVersionRoot != null)
{
DirectoryReference InTreeSDKVersionPath = DirectoryReference.Combine(InTreeSDKVersionRoot, ExpectedSDKVersion);
if (DirectoryReference.Exists(InTreeSDKVersionPath))
{
MultiArchRoot = InTreeSDKVersionPath.FullName;
}
}
}
return MultiArchRoot;
}
///
/// Returns the SDK path for a specific architecture
/// WARNING: Do not cache this value - it may be changed after sourcing OutputEnvVars.txt
///
/// Valid SDK string
public virtual string GetBaseLinuxPathForArchitecture(string Architecture)
{
// if new multi-arch toolchain is used, prefer it
string MultiArchRoot = GetSDKLocation();
string BaseLinuxPath;
if (!String.IsNullOrEmpty(MultiArchRoot))
{
BaseLinuxPath = Path.Combine(MultiArchRoot, Architecture);
}
else
{
// use cross linux toolchain if LINUX_ROOT is specified
BaseLinuxPath = Environment.GetEnvironmentVariable("LINUX_ROOT");
}
return BaseLinuxPath;
}
///
/// Whether the path contains a valid clang version
///
private static bool IsValidClangPath(DirectoryReference BaseLinuxPath)
{
FileReference ClangPath = FileReference.Combine(BaseLinuxPath, @"bin", (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win64) ? "clang++.exe" : "clang++");
return FileReference.Exists(ClangPath);
}
///
/// Whether the required external SDKs are installed for this platform
///
protected override SDKStatus HasRequiredManualSDKInternal()
{
// FIXME: UBT should loop across all the architectures and compile for all the selected ones.
// do not cache this value - it may be changed after sourcing OutputEnvVars.txt
string BaseLinuxPath = GetBaseLinuxPathForArchitecture(LinuxPlatform.DefaultArchitecture);
// BaseLinuxPath is specified
if (!String.IsNullOrEmpty(BaseLinuxPath))
{
// paths to our toolchains
BaseLinuxPath = BaseLinuxPath.Replace("\"", "");
if (IsValidClangPath(new DirectoryReference(BaseLinuxPath)))
{
return SDKStatus.Valid;
}
}
if (CanUseSystemCompiler())
{
if (!String.IsNullOrEmpty(LinuxCommon.WhichClang()) || !String.IsNullOrEmpty(LinuxCommon.WhichGcc()))
{
return SDKStatus.Valid;
}
}
return SDKStatus.Invalid;
}
}
class LinuxPlatformFactory : UEBuildPlatformFactory
{
protected override UnrealTargetPlatform TargetPlatform
{
get { return UnrealTargetPlatform.Linux; }
}
///
/// Register the platform with the UEBuildPlatform class
///
protected override void RegisterBuildPlatforms(SDKOutputLevel OutputLevel)
{
LinuxPlatformSDK SDK = new LinuxPlatformSDK();
SDK.ManageAndValidateSDK(OutputLevel);
if ((ProjectFileGenerator.bGenerateProjectFiles == true) || (SDK.HasRequiredSDKsInstalled() == SDKStatus.Valid))
{
FileReference LinuxTargetPlatformFile = FileReference.Combine(UnrealBuildTool.EngineSourceDirectory, "Developer", "Linux", "LinuxTargetPlatform", "LinuxTargetPlatform.Build.cs");
if (FileReference.Exists(LinuxTargetPlatformFile))
{
// Register this build platform for Linux
Log.TraceVerbose(" Registering for {0}", UnrealTargetPlatform.Linux.ToString());
UEBuildPlatform.RegisterBuildPlatform(new LinuxPlatform(SDK));
UEBuildPlatform.RegisterPlatformWithGroup(UnrealTargetPlatform.Linux, UnrealPlatformGroup.Unix);
}
}
}
}
}