2020-01-07 13:45:01 -05:00
// Copyright Epic Games, Inc. All Rights Reserved.
2018-08-16 13:53:43 -04:00
using System ;
using System.Collections ;
using System.Collections.Generic ;
using System.IO ;
using System.Diagnostics ;
using System.Linq ;
using System.Security.AccessControl ;
using System.Xml ;
using System.Text ;
using System.Text.RegularExpressions ;
2020-12-21 23:07:37 -04:00
using EpicGames.Core ;
2018-08-16 13:53:43 -04:00
using System.Security.Cryptography.X509Certificates ;
2021-06-09 12:54:42 -04:00
using UnrealBuildBase ;
2018-08-16 13:53:43 -04:00
2020-12-20 18:47:42 -04:00
#nullable disable
2018-08-16 13:53:43 -04:00
namespace UnrealBuildTool
{
/// <summary>
/// Stores information about how a local directory maps to a remote directory
/// </summary>
[DebuggerDisplay("{LocalDirectory}")]
class RemoteMapping
{
public DirectoryReference LocalDirectory ;
public string RemoteDirectory ;
public RemoteMapping ( DirectoryReference LocalDirectory , string RemoteDirectory )
{
this . LocalDirectory = LocalDirectory ;
this . RemoteDirectory = RemoteDirectory ;
}
}
/// <summary>
/// Handles uploading and building on a remote Mac
/// </summary>
class RemoteMac
{
/// <summary>
2019-10-11 16:59:16 -04:00
/// These two variables will be loaded from the XML config file in XmlConfigLoader.Init().
2018-08-16 13:53:43 -04:00
/// </summary>
[XmlConfigFile]
private readonly string ServerName ;
/// <summary>
2019-10-11 16:59:16 -04:00
/// The remote username.
2018-08-16 13:53:43 -04:00
/// </summary>
[XmlConfigFile]
private readonly string UserName ;
/// <summary>
2019-10-11 16:59:16 -04:00
/// If set, instead of looking for RemoteToolChainPrivate.key in the usual places (Documents/Unreal, Engine/UnrealBuildTool/SSHKeys or Engine/Build/SSHKeys), this private key will be used.
2018-08-16 13:53:43 -04:00
/// </summary>
[XmlConfigFile]
private FileReference SshPrivateKey ;
/// <summary>
2019-10-11 16:59:16 -04:00
/// The authentication used for Rsync (for the -e rsync flag).
2018-08-16 13:53:43 -04:00
/// </summary>
[XmlConfigFile]
2020-01-21 06:14:35 -05:00
private string RsyncAuthentication = "./ssh -i '${CYGWIN_SSH_PRIVATE_KEY}'" ;
2018-08-16 13:53:43 -04:00
/// <summary>
2019-10-11 16:59:16 -04:00
/// The authentication used for SSH (probably similar to RsyncAuthentication).
2018-08-16 13:53:43 -04:00
/// </summary>
[XmlConfigFile]
private string SshAuthentication = "-i '${CYGWIN_SSH_PRIVATE_KEY}'" ;
/// <summary>
/// Save the specified port so that RemoteServerName is the machine address only
/// </summary>
private readonly int ServerPort = 22 ; // Default ssh port
/// <summary>
/// Path to Rsync
/// </summary>
private readonly FileReference RsyncExe ;
/// <summary>
/// Path to SSH
/// </summary>
private readonly FileReference SshExe ;
/// <summary>
/// The project being built. Settings will be read from config files in this project.
/// </summary>
private readonly FileReference ProjectFile ;
/// <summary>
2020-01-31 10:34:10 -05:00
/// The project descriptor for the project being built.
2018-08-16 13:53:43 -04:00
/// </summary>
2020-01-31 10:34:10 -05:00
private readonly ProjectDescriptor ProjectDescriptor ;
/// <summary>
/// A set of directories containing additional paths to be built.
/// </summary>
private readonly List < DirectoryReference > AdditionalPaths ;
2018-08-16 13:53:43 -04:00
/// <summary>
/// The base directory on the remote machine
/// </summary>
private string RemoteBaseDir ;
/// <summary>
/// Mappings from local directories to remote directories
/// </summary>
private List < RemoteMapping > Mappings ;
/// <summary>
/// Arguments that are used by every Ssh call
/// </summary>
private List < string > CommonSshArguments ;
/// <summary>
/// Arguments that are used by every Rsync call
/// </summary>
2018-10-04 21:12:36 -04:00
private List < string > BasicRsyncArguments ;
/// <summary>
/// Arguments that are used by directory Rsync call
/// </summary>
2018-08-16 13:53:43 -04:00
private List < string > CommonRsyncArguments ;
2019-03-28 12:09:05 -04:00
private string IniBundleIdentifier = "" ;
2018-08-16 13:53:43 -04:00
/// <summary>
/// Constructor
/// </summary>
/// <param name="ProjectFile">Project to read settings from</param>
public RemoteMac ( FileReference ProjectFile )
{
2021-06-11 18:20:44 -04:00
this . RsyncExe = FileReference . Combine ( Unreal . EngineDirectory , "Extras" , "ThirdPartyNotUE" , "cwrsync" , "bin" , "rsync.exe" ) ;
this . SshExe = FileReference . Combine ( Unreal . EngineDirectory , "Extras" , "ThirdPartyNotUE" , "cwrsync" , "bin" , "ssh.exe" ) ;
2018-08-16 13:53:43 -04:00
this . ProjectFile = ProjectFile ;
2020-01-31 10:34:10 -05:00
if ( ProjectFile ! = null )
{
this . ProjectDescriptor = ProjectDescriptor . FromFile ( ProjectFile ) ;
this . AdditionalPaths = new List < DirectoryReference > ( ) ;
this . ProjectDescriptor . AddAdditionalPaths ( this . AdditionalPaths , ProjectFile . Directory ) ;
if ( this . AdditionalPaths . Count = = 0 )
{
this . AdditionalPaths = null ;
}
}
2018-08-16 13:53:43 -04:00
// Apply settings from the XML file
XmlConfig . ApplyTo ( this ) ;
// Get the project config file path
DirectoryReference EngineIniPath = ProjectFile ! = null ? ProjectFile . Directory : null ;
if ( EngineIniPath = = null & & UnrealBuildTool . GetRemoteIniPath ( ) ! = null )
{
EngineIniPath = new DirectoryReference ( UnrealBuildTool . GetRemoteIniPath ( ) ) ;
}
ConfigHierarchy Ini = ConfigCache . ReadHierarchy ( ConfigHierarchyType . Engine , EngineIniPath , UnrealTargetPlatform . IOS ) ;
// Read the project settings if we don't have anything in the build configuration settings
if ( String . IsNullOrEmpty ( ServerName ) )
{
// Read the server name
string IniServerName ;
if ( Ini . GetString ( "/Script/IOSRuntimeSettings.IOSRuntimeSettings" , "RemoteServerName" , out IniServerName ) & & ! String . IsNullOrEmpty ( IniServerName ) )
{
this . ServerName = IniServerName ;
}
else
{
throw new BuildException ( "Remote compiling requires a server name. Use the editor (Project Settings > IOS) to set up your remote compilation settings." ) ;
}
// Parse the username
string IniUserName ;
if ( Ini . GetString ( "/Script/IOSRuntimeSettings.IOSRuntimeSettings" , "RSyncUsername" , out IniUserName ) & & ! String . IsNullOrEmpty ( IniUserName ) )
{
this . UserName = IniUserName ;
}
}
// Split port out from the server name
int PortIdx = ServerName . LastIndexOf ( ':' ) ;
if ( PortIdx ! = - 1 )
{
string Port = ServerName . Substring ( PortIdx + 1 ) ;
if ( ! int . TryParse ( Port , out ServerPort ) )
{
throw new BuildException ( "Unable to parse port number from '{0}'" , ServerName ) ;
}
ServerName = ServerName . Substring ( 0 , PortIdx ) ;
}
// If a user name is not set, use the current user
if ( String . IsNullOrEmpty ( UserName ) )
{
UserName = Environment . UserName ;
}
// Print out the server info
Log . TraceInformation ( "[Remote] Using remote server '{0}' on port {1} (user '{2}')" , ServerName , ServerPort , UserName ) ;
// Get the path to the SSH private key
string OverrideSshPrivateKeyPath ;
if ( Ini . GetString ( "/Script/IOSRuntimeSettings.IOSRuntimeSettings" , "SSHPrivateKeyOverridePath" , out OverrideSshPrivateKeyPath ) & & ! String . IsNullOrEmpty ( OverrideSshPrivateKeyPath ) )
{
SshPrivateKey = new FileReference ( OverrideSshPrivateKeyPath ) ;
if ( ! FileReference . Exists ( SshPrivateKey ) )
{
throw new BuildException ( "SSH private key specified in config file ({0}) does not exist." , SshPrivateKey ) ;
}
}
2019-03-28 12:09:05 -04:00
Ini . GetString ( "/Script/IOSRuntimeSettings.IOSRuntimeSettings" , "BundleIdentifier" , out IniBundleIdentifier ) ;
2018-08-16 13:53:43 -04:00
// If it's not set, look in the standard locations. If that fails, spawn the batch file to generate one.
if ( SshPrivateKey = = null & & ! TryGetSshPrivateKey ( out SshPrivateKey ) )
{
Log . TraceWarning ( "No SSH private key found for {0}@{1}. Launching SSH to generate one." , UserName , ServerName ) ;
StringBuilder CommandLine = new StringBuilder ( ) ;
2021-06-11 18:20:44 -04:00
CommandLine . AppendFormat ( "/C \"\"{0}\"" , FileReference . Combine ( Unreal . EngineDirectory , "Build" , "BatchFiles" , "MakeAndInstallSSHKey.bat" ) ) ;
2018-08-16 13:53:43 -04:00
CommandLine . AppendFormat ( " \"{0}\"" , SshExe ) ;
CommandLine . AppendFormat ( " \"{0}\"" , ServerPort ) ;
CommandLine . AppendFormat ( " \"{0}\"" , RsyncExe ) ;
CommandLine . AppendFormat ( " \"{0}\"" , UserName ) ;
CommandLine . AppendFormat ( " \"{0}\"" , ServerName ) ;
CommandLine . AppendFormat ( " \"{0}\"" , DirectoryReference . GetSpecialFolder ( Environment . SpecialFolder . MyDocuments ) ) ;
CommandLine . AppendFormat ( " \"{0}\"" , GetLocalCygwinPath ( DirectoryReference . GetSpecialFolder ( Environment . SpecialFolder . MyDocuments ) ) ) ;
2021-06-11 18:20:44 -04:00
CommandLine . AppendFormat ( " \"{0}\"" , Unreal . EngineDirectory ) ;
2018-08-16 13:53:43 -04:00
CommandLine . Append ( "\"" ) ;
2019-01-14 12:11:24 -05:00
using ( Process ChildProcess = Process . Start ( BuildHostPlatform . Current . Shell . FullName , CommandLine . ToString ( ) ) )
2018-08-16 13:53:43 -04:00
{
ChildProcess . WaitForExit ( ) ;
}
if ( ! TryGetSshPrivateKey ( out SshPrivateKey ) )
{
throw new BuildException ( "Failed to generate SSH private key for {0}@{1}." , UserName , ServerName ) ;
}
}
2018-11-09 13:09:06 -05:00
// Print the path to the private key
Log . TraceInformation ( "[Remote] Using private key at {0}" , SshPrivateKey ) ;
2018-08-16 13:53:43 -04:00
// resolve the rest of the strings
RsyncAuthentication = ExpandVariables ( RsyncAuthentication ) ;
SshAuthentication = ExpandVariables ( SshAuthentication ) ;
// Build a list of arguments for SSH
CommonSshArguments = new List < string > ( ) ;
CommonSshArguments . Add ( "-o BatchMode=yes" ) ;
CommonSshArguments . Add ( SshAuthentication ) ;
CommonSshArguments . Add ( String . Format ( "-p {0}" , ServerPort ) ) ;
CommonSshArguments . Add ( String . Format ( "\"{0}@{1}\"" , UserName , ServerName ) ) ;
// Build a list of arguments for Rsync
2018-10-04 21:12:36 -04:00
BasicRsyncArguments = new List < string > ( ) ;
BasicRsyncArguments . Add ( "--compress" ) ;
BasicRsyncArguments . Add ( "--verbose" ) ;
BasicRsyncArguments . Add ( String . Format ( "--rsh=\"{0} -p {1}\"" , RsyncAuthentication , ServerPort ) ) ;
2018-10-16 14:21:36 -04:00
BasicRsyncArguments . Add ( "--chmod=ugo=rwx" ) ;
2018-10-04 21:12:36 -04:00
// Build a list of arguments for Rsync filters
CommonRsyncArguments = new List < string > ( BasicRsyncArguments ) ;
2018-08-16 13:53:43 -04:00
CommonRsyncArguments . Add ( "--recursive" ) ;
CommonRsyncArguments . Add ( "--delete" ) ; // Delete anything not in the source directory
CommonRsyncArguments . Add ( "--delete-excluded" ) ; // Delete anything not in the source directory
CommonRsyncArguments . Add ( "--times" ) ; // Preserve modification times
2019-01-14 12:11:24 -05:00
CommonRsyncArguments . Add ( "--omit-dir-times" ) ; // Ignore modification times for directories
2018-10-04 21:12:36 -04:00
CommonRsyncArguments . Add ( "--prune-empty-dirs" ) ; // Remove empty directories from the file list
2018-10-03 09:21:19 -04:00
// Get the remote base directory
2021-02-10 15:21:03 -04:00
string RemoteServerOverrideBuildPath ;
if ( Ini . GetString ( "/Script/IOSRuntimeSettings.IOSRuntimeSettings" , "RemoteServerOverrideBuildPath" , out RemoteServerOverrideBuildPath ) & & ! String . IsNullOrEmpty ( RemoteServerOverrideBuildPath ) )
2018-10-03 09:21:19 -04:00
{
2021-02-10 15:21:03 -04:00
RemoteBaseDir = String . Format ( "{0}/{1}" , RemoteServerOverrideBuildPath . Trim ( ) . TrimEnd ( '/' ) , Environment . MachineName ) ;
2018-10-03 09:21:19 -04:00
}
2021-02-10 15:21:03 -04:00
else
{
StringBuilder Output ;
if ( ExecuteAndCaptureOutput ( "'echo ~'" , out Output ) ! = 0 )
{
throw new BuildException ( "Unable to determine home directory for remote user. SSH output:\n{0}" , StringUtils . Indent ( Output . ToString ( ) , " " ) ) ;
}
RemoteBaseDir = String . Format ( "{0}/UE5/Builds/{1}" , Output . ToString ( ) . Trim ( ) . TrimEnd ( '/' ) , Environment . MachineName ) ;
}
2018-10-03 09:21:19 -04:00
Log . TraceInformation ( "[Remote] Using base directory '{0}'" , RemoteBaseDir ) ;
// Build the list of directory mappings between the local and remote machines
Mappings = new List < RemoteMapping > ( ) ;
2021-06-11 18:20:44 -04:00
Mappings . Add ( new RemoteMapping ( Unreal . EngineDirectory , GetRemotePath ( Unreal . EngineDirectory ) ) ) ;
if ( ProjectFile ! = null & & ! ProjectFile . IsUnderDirectory ( Unreal . EngineDirectory ) )
2018-10-03 09:21:19 -04:00
{
Mappings . Add ( new RemoteMapping ( ProjectFile . Directory , GetRemotePath ( ProjectFile . Directory ) ) ) ;
}
2020-01-31 10:34:10 -05:00
if ( AdditionalPaths ! = null & & ProjectFile ! = null )
{
foreach ( DirectoryReference AdditionalPath in AdditionalPaths )
{
2021-06-11 18:20:44 -04:00
if ( ! AdditionalPath . IsUnderDirectory ( Unreal . EngineDirectory ) & &
2020-01-31 10:34:10 -05:00
! AdditionalPath . IsUnderDirectory ( ProjectFile . Directory ) )
{
Mappings . Add ( new RemoteMapping ( AdditionalPath , GetRemotePath ( AdditionalPath ) ) ) ;
}
}
}
2018-08-16 13:53:43 -04:00
}
/// <summary>
/// Attempts to get the SSH private key from the standard locations
/// </summary>
/// <param name="OutPrivateKey">If successful, receives the location of the private key that was found</param>
/// <returns>True if a private key was found, false otherwise</returns>
private bool TryGetSshPrivateKey ( out FileReference OutPrivateKey )
{
// Build a list of all the places to look for a private key
List < DirectoryReference > Locations = new List < DirectoryReference > ( ) ;
Locations . Add ( DirectoryReference . Combine ( DirectoryReference . GetSpecialFolder ( Environment . SpecialFolder . ApplicationData ) , "Unreal Engine" , "UnrealBuildTool" ) ) ;
Locations . Add ( DirectoryReference . Combine ( DirectoryReference . GetSpecialFolder ( Environment . SpecialFolder . Personal ) , "Unreal Engine" , "UnrealBuildTool" ) ) ;
if ( ProjectFile ! = null )
{
2020-04-11 08:47:06 -04:00
Locations . Add ( DirectoryReference . Combine ( ProjectFile . Directory , "Restricted" , "NotForLicensees" , "Build" ) ) ;
Locations . Add ( DirectoryReference . Combine ( ProjectFile . Directory , "Restricted" , "NoRedist" , "Build" ) ) ;
2018-08-16 13:53:43 -04:00
Locations . Add ( DirectoryReference . Combine ( ProjectFile . Directory , "Build" ) ) ;
}
2021-06-11 18:20:44 -04:00
Locations . Add ( DirectoryReference . Combine ( Unreal . EngineDirectory , "Restricted" , "NotForLicensees" , "Build" ) ) ;
Locations . Add ( DirectoryReference . Combine ( Unreal . EngineDirectory , "Restricted" , "NoRedist" , "Build" ) ) ;
Locations . Add ( DirectoryReference . Combine ( Unreal . EngineDirectory , "Build" ) ) ;
2018-08-16 13:53:43 -04:00
// Find the first that exists
foreach ( DirectoryReference Location in Locations )
{
FileReference KeyFile = FileReference . Combine ( Location , "SSHKeys" , ServerName , UserName , "RemoteToolChainPrivate.key" ) ;
if ( FileReference . Exists ( KeyFile ) )
{
2018-11-09 13:09:06 -05:00
// MacOS Mojave includes a new version of SSH that generates keys that are incompatible with our version of SSH. Make sure the detected keys have the right signature.
string Text = FileReference . ReadAllText ( KeyFile ) ;
if ( Text . Contains ( "---BEGIN RSA PRIVATE KEY---" ) )
{
OutPrivateKey = KeyFile ;
return true ;
}
2018-08-16 13:53:43 -04:00
}
}
// Nothing found
OutPrivateKey = null ;
return false ;
}
/// <summary>
/// Expand all the variables in the given string
/// </summary>
/// <param name="Input">The input string</param>
/// <returns>String with any variables expanded</returns>
private string ExpandVariables ( string Input )
{
string Result = Input ;
Result = Result . Replace ( "${SSH_PRIVATE_KEY}" , SshPrivateKey . FullName ) ;
Result = Result . Replace ( "${CYGWIN_SSH_PRIVATE_KEY}" , GetLocalCygwinPath ( SshPrivateKey ) ) ;
return Result ;
}
/// <summary>
/// Flush the remote machine, removing all existing files
/// </summary>
public void FlushRemote ( )
{
Log . TraceInformation ( "[Remote] Deleting all files under {0}..." , RemoteBaseDir ) ;
Execute ( "/" , String . Format ( "rm -rf \"{0}\"" , RemoteBaseDir ) ) ;
}
2019-01-14 12:11:24 -05:00
/// <summary>
/// Returns true if the remote executor supports this target platform
/// </summary>
/// <param name="Platform">The platform to check</param>
/// <returns>True if the remote mac handles this target platform</returns>
public static bool HandlesTargetPlatform ( UnrealTargetPlatform Platform )
{
return BuildHostPlatform . Current . Platform = = UnrealTargetPlatform . Win64 & & ( Platform = = UnrealTargetPlatform . Mac | | Platform = = UnrealTargetPlatform . IOS | | Platform = = UnrealTargetPlatform . TVOS ) ;
}
/// <summary>
/// Clean a target remotely
/// </summary>
/// <param name="TargetDesc">Descriptor for the target to build</param>
/// <returns>True if the build succeeded, false otherwise</returns>
public bool Clean ( TargetDescriptor TargetDesc )
{
// Translate all the arguments for the remote
List < string > RemoteArguments = GetRemoteArgumentsForTarget ( TargetDesc , null ) ;
RemoteArguments . Add ( "-Clean" ) ;
// Upload the workspace
DirectoryReference TempDir = CreateTempDirectory ( TargetDesc ) ;
UploadWorkspace ( TempDir ) ;
// Execute the compile
Log . TraceInformation ( "[Remote] Executing clean..." ) ;
StringBuilder BuildCommandLine = new StringBuilder ( "Engine/Build/BatchFiles/Mac/Build.sh" ) ;
foreach ( string RemoteArgument in RemoteArguments )
{
BuildCommandLine . AppendFormat ( " {0}" , EscapeShellArgument ( RemoteArgument ) ) ;
}
2021-06-11 18:20:44 -04:00
int Result = Execute ( GetRemotePath ( Unreal . RootDirectory ) , BuildCommandLine . ToString ( ) ) ;
2019-01-14 12:11:24 -05:00
return Result = = 0 ;
}
2018-08-16 13:53:43 -04:00
/// <summary>
/// Build a target remotely
/// </summary>
/// <param name="TargetDesc">Descriptor for the target to build</param>
/// <param name="RemoteLogFile">Path to store the remote log file</param>
2020-03-12 14:08:52 -04:00
/// <param name="bSkipPreBuildTargets">If true then any PreBuildTargets will be skipped</param>
2018-08-16 13:53:43 -04:00
/// <returns>True if the build succeeded, false otherwise</returns>
2020-03-12 14:08:52 -04:00
public bool Build ( TargetDescriptor TargetDesc , FileReference RemoteLogFile , bool bSkipPreBuildTargets )
2018-08-16 13:53:43 -04:00
{
2020-03-12 14:08:52 -04:00
// Compile the rules assembly
RulesAssembly RulesAssembly = RulesCompiler . CreateTargetRulesAssembly ( TargetDesc . ProjectFile , TargetDesc . Name , false , false , TargetDesc . ForeignPlugin ) ;
// Create the target rules
TargetRules Rules = RulesAssembly . CreateTargetRules ( TargetDesc . Name , TargetDesc . Platform , TargetDesc . Configuration , TargetDesc . Architecture , TargetDesc . ProjectFile , TargetDesc . AdditionalArguments ) ;
if ( ! bSkipPreBuildTargets )
{
foreach ( TargetInfo PreBuildTargetInfo in Rules . PreBuildTargets )
{
RemoteMac PreBuildTargetRemoteMac = new RemoteMac ( ProjectFile ) ;
TargetDescriptor PreBuildTargetDesc = new TargetDescriptor ( PreBuildTargetInfo . ProjectFile , PreBuildTargetInfo . Name , PreBuildTargetInfo . Platform , PreBuildTargetInfo . Configuration , PreBuildTargetInfo . Architecture , PreBuildTargetInfo . Arguments ) ;
Log . TraceInformation ( "[Remote] Building pre target [{0}] for [{1}] " , PreBuildTargetDesc . ToString ( ) , TargetDesc . ToString ( ) ) ;
if ( ! PreBuildTargetRemoteMac . Build ( PreBuildTargetDesc , RemoteLogFile , false ) )
{
return false ;
}
}
}
2018-08-16 13:53:43 -04:00
// Get the directory for working files
2019-01-14 12:11:24 -05:00
DirectoryReference TempDir = CreateTempDirectory ( TargetDesc ) ;
2018-08-16 13:53:43 -04:00
2019-01-14 12:11:24 -05:00
// Map the path containing the remote log file
2018-10-22 04:40:58 -04:00
bool bLogIsMapped = false ;
foreach ( RemoteMapping Mapping in Mappings )
{
if ( RemoteLogFile . Directory . FullName . Equals ( Mapping . LocalDirectory . FullName , StringComparison . InvariantCultureIgnoreCase ) )
{
bLogIsMapped = true ;
break ;
}
}
if ( ! bLogIsMapped )
{
Mappings . Add ( new RemoteMapping ( RemoteLogFile . Directory , GetRemotePath ( RemoteLogFile . Directory ) ) ) ;
}
2018-08-16 13:53:43 -04:00
// Path to the local manifest file. This has to be translated from the remote format after the build is complete.
List < FileReference > LocalManifestFiles = new List < FileReference > ( ) ;
// Path to the remote manifest file
FileReference RemoteManifestFile = FileReference . Combine ( TempDir , "Manifest.xml" ) ;
// Prepare the arguments we will pass to the remote build
2019-01-14 12:11:24 -05:00
List < string > RemoteArguments = GetRemoteArgumentsForTarget ( TargetDesc , LocalManifestFiles ) ;
2018-08-16 13:53:43 -04:00
RemoteArguments . Add ( String . Format ( "-Log={0}" , GetRemotePath ( RemoteLogFile ) ) ) ;
RemoteArguments . Add ( String . Format ( "-Manifest={0}" , GetRemotePath ( RemoteManifestFile ) ) ) ;
2020-03-12 14:08:52 -04:00
RemoteArguments . Add ( String . Format ( "-SkipPreBuildTargets" ) ) ;
2018-08-16 13:53:43 -04:00
// Handle any per-platform setup that is required
if ( TargetDesc . Platform = = UnrealTargetPlatform . IOS | | TargetDesc . Platform = = UnrealTargetPlatform . TVOS )
{
// Always generate a .stub
RemoteArguments . Add ( "-CreateStub" ) ;
2018-10-16 14:21:36 -04:00
// Cannot use makefiles, since we need PostBuildSync() to generate the IPA (and that requires a TargetRules instance)
RemoteArguments . Add ( "-NoUBTMakefiles" ) ;
2018-08-16 13:53:43 -04:00
// Get the provisioning data for this project
2019-03-28 12:09:05 -04:00
IOSProvisioningData ProvisioningData = ( ( IOSPlatform ) UEBuildPlatform . GetBuildPlatform ( TargetDesc . Platform ) ) . ReadProvisioningData ( TargetDesc . ProjectFile , TargetDesc . AdditionalArguments . HasOption ( "-distribution" ) , IniBundleIdentifier ) ;
2018-08-16 13:53:43 -04:00
if ( ProvisioningData = = null | | ProvisioningData . MobileProvisionFile = = null )
{
throw new BuildException ( "Unable to find mobile provision for {0}. See log for more information." , TargetDesc . Name ) ;
}
// Create a local copy of the provision
FileReference MobileProvisionFile = FileReference . Combine ( TempDir , ProvisioningData . MobileProvisionFile . GetFileName ( ) ) ;
if ( FileReference . Exists ( MobileProvisionFile ) )
{
FileReference . SetAttributes ( MobileProvisionFile , FileAttributes . Normal ) ;
}
FileReference . Copy ( ProvisioningData . MobileProvisionFile , MobileProvisionFile , true ) ;
Log . TraceInformation ( "[Remote] Uploading {0}" , MobileProvisionFile ) ;
UploadFile ( MobileProvisionFile ) ;
2018-09-28 16:50:29 -04:00
// Extract the certificate for the project. Try to avoid calling IPP if we already have it.
2018-08-16 13:53:43 -04:00
FileReference CertificateFile = FileReference . Combine ( TempDir , "Certificate.p12" ) ;
2018-09-28 16:50:29 -04:00
FileReference CertificateInfoFile = FileReference . Combine ( TempDir , "Certificate.txt" ) ;
string CertificateInfoContents = String . Format ( "{0}\n{1}" , ProvisioningData . MobileProvisionFile , FileReference . GetLastWriteTimeUtc ( ProvisioningData . MobileProvisionFile ) . Ticks ) ;
if ( ! FileReference . Exists ( CertificateFile ) | | ! FileReference . Exists ( CertificateInfoFile ) | | FileReference . ReadAllText ( CertificateInfoFile ) ! = CertificateInfoContents )
2018-08-16 13:53:43 -04:00
{
2018-09-28 16:50:29 -04:00
Log . TraceInformation ( "[Remote] Exporting certificate for {0}..." , ProvisioningData . MobileProvisionFile ) ;
2018-08-16 13:53:43 -04:00
StringBuilder Arguments = new StringBuilder ( "ExportCertificate" ) ;
if ( TargetDesc . ProjectFile = = null )
{
Arguments . AppendFormat ( " \"{0}\"" , UnrealBuildTool . EngineSourceDirectory ) ;
}
else
{
Arguments . AppendFormat ( " \"{0}\"" , TargetDesc . ProjectFile . Directory ) ;
}
2018-09-28 16:50:29 -04:00
Arguments . AppendFormat ( " -provisionfile \"{0}\"" , ProvisioningData . MobileProvisionFile ) ;
Arguments . AppendFormat ( " -outputcertificate \"{0}\"" , CertificateFile ) ;
2018-09-10 09:49:20 -04:00
if ( TargetDesc . Platform = = UnrealTargetPlatform . TVOS )
{
Arguments . Append ( " -tvos" ) ;
}
2018-08-16 13:53:43 -04:00
ProcessStartInfo StartInfo = new ProcessStartInfo ( ) ;
2021-06-11 18:20:44 -04:00
StartInfo . FileName = FileReference . Combine ( Unreal . EngineDirectory , "Binaries" , "DotNET" , "IOS" , "IPhonePackager.exe" ) . FullName ;
2018-08-16 13:53:43 -04:00
StartInfo . Arguments = Arguments . ToString ( ) ;
if ( Utils . RunLocalProcessAndLogOutput ( StartInfo ) ! = 0 )
{
throw new BuildException ( "IphonePackager failed." ) ;
}
2018-09-28 16:50:29 -04:00
FileReference . WriteAllText ( CertificateInfoFile , CertificateInfoContents ) ;
2018-08-16 13:53:43 -04:00
}
// Upload the certificate to the remote
Log . TraceInformation ( "[Remote] Uploading {0}" , CertificateFile ) ;
UploadFile ( CertificateFile ) ;
// Tell the remote UBT instance to use them
RemoteArguments . Add ( String . Format ( "-ImportProvision={0}" , GetRemotePath ( MobileProvisionFile ) ) ) ;
RemoteArguments . Add ( String . Format ( "-ImportCertificate={0}" , GetRemotePath ( CertificateFile ) ) ) ;
RemoteArguments . Add ( String . Format ( "-ImportCertificatePassword=A" ) ) ;
}
// Upload the workspace files
UploadWorkspace ( TempDir ) ;
// Execute the compile
Log . TraceInformation ( "[Remote] Executing build" ) ;
StringBuilder BuildCommandLine = new StringBuilder ( "Engine/Build/BatchFiles/Mac/Build.sh" ) ;
foreach ( string RemoteArgument in RemoteArguments )
{
BuildCommandLine . AppendFormat ( " {0}" , EscapeShellArgument ( RemoteArgument ) ) ;
}
2021-06-11 18:20:44 -04:00
int Result = Execute ( GetRemotePath ( Unreal . RootDirectory ) , BuildCommandLine . ToString ( ) ) ;
2018-08-16 13:53:43 -04:00
if ( Result ! = 0 )
{
if ( RemoteLogFile ! = null )
{
Log . TraceInformation ( "[Remote] Downloading {0}" , RemoteLogFile ) ;
DownloadFile ( RemoteLogFile ) ;
}
return false ;
}
// Download the manifest
Log . TraceInformation ( "[Remote] Downloading {0}" , RemoteManifestFile ) ;
DownloadFile ( RemoteManifestFile ) ;
// Convert the manifest to local form
BuildManifest Manifest = Utils . ReadClass < BuildManifest > ( RemoteManifestFile . FullName ) ;
for ( int Idx = 0 ; Idx < Manifest . BuildProducts . Count ; Idx + + )
{
Manifest . BuildProducts [ Idx ] = GetLocalPath ( Manifest . BuildProducts [ Idx ] ) . FullName ;
}
// Download the files from the remote
2019-01-14 12:11:24 -05:00
Log . TraceInformation ( "[Remote] Downloading build products" ) ;
2018-08-16 13:53:43 -04:00
2019-01-14 12:11:24 -05:00
List < FileReference > FilesToDownload = new List < FileReference > ( ) ;
FilesToDownload . Add ( RemoteLogFile ) ;
FilesToDownload . AddRange ( Manifest . BuildProducts . Select ( x = > new FileReference ( x ) ) ) ;
DownloadFiles ( FilesToDownload ) ;
2018-08-16 13:53:43 -04:00
2019-02-12 09:39:55 -05:00
// Copy remote FrameworkAssets directory as it could contain resource bundles that must be packaged locally.
2021-06-11 18:20:44 -04:00
DirectoryReference BaseDir = DirectoryReference . FromFile ( TargetDesc . ProjectFile ) ? ? Unreal . EngineDirectory ;
2019-02-28 10:12:36 -05:00
DirectoryReference FrameworkAssetsDir = DirectoryReference . Combine ( BaseDir , "Intermediate" , TargetDesc . Platform = = UnrealTargetPlatform . IOS ? "IOS" : "TVOS" , "FrameworkAssets" ) ;
2019-04-14 09:07:30 -04:00
if ( RemoteDirectoryExists ( FrameworkAssetsDir ) )
{
Log . TraceInformation ( "[Remote] Downloading {0}" , FrameworkAssetsDir ) ;
DownloadDirectory ( FrameworkAssetsDir ) ;
}
2019-02-12 09:39:55 -05:00
2018-08-16 13:53:43 -04:00
// Write out all the local manifests
foreach ( FileReference LocalManifestFile in LocalManifestFiles )
{
Log . TraceInformation ( "[Remote] Writing {0}" , LocalManifestFile ) ;
Utils . WriteClass < BuildManifest > ( Manifest , LocalManifestFile . FullName , "" ) ;
}
2019-02-12 09:39:55 -05:00
2018-08-16 13:53:43 -04:00
return true ;
}
2019-01-14 12:11:24 -05:00
/// <summary>
/// Creates a temporary directory for the given target
/// </summary>
/// <param name="TargetDesc">The target descriptor</param>
/// <returns>Directory to use for temporary files</returns>
static DirectoryReference CreateTempDirectory ( TargetDescriptor TargetDesc )
{
2021-06-11 18:20:44 -04:00
DirectoryReference BaseDir = DirectoryReference . FromFile ( TargetDesc . ProjectFile ) ? ? Unreal . EngineDirectory ;
2019-01-14 12:11:24 -05:00
DirectoryReference TempDir = DirectoryReference . Combine ( BaseDir , "Intermediate" , "Remote" , TargetDesc . Name , TargetDesc . Platform . ToString ( ) , TargetDesc . Configuration . ToString ( ) ) ;
DirectoryReference . CreateDirectory ( TempDir ) ;
return TempDir ;
}
/// <summary>
/// Translate the arguments for a target descriptor for the remote machine
/// </summary>
/// <param name="TargetDesc">The target descriptor</param>
/// <param name="LocalManifestFiles">Manifest files to be output from this target</param>
/// <return>List of remote arguments</return>
List < string > GetRemoteArgumentsForTarget ( TargetDescriptor TargetDesc , List < FileReference > LocalManifestFiles )
{
List < string > RemoteArguments = new List < string > ( ) ;
RemoteArguments . Add ( TargetDesc . Name ) ;
RemoteArguments . Add ( TargetDesc . Platform . ToString ( ) ) ;
RemoteArguments . Add ( TargetDesc . Configuration . ToString ( ) ) ;
RemoteArguments . Add ( "-SkipRulesCompile" ) ; // Use the rules assembly built locally
RemoteArguments . Add ( String . Format ( "-XmlConfigCache={0}" , GetRemotePath ( XmlConfig . CacheFile ) ) ) ; // Use the XML config cache built locally, since the remote won't have it
2019-04-14 07:39:41 -04:00
string RemoteIniPath = UnrealBuildTool . GetRemoteIniPath ( ) ;
if ( ! String . IsNullOrEmpty ( RemoteIniPath ) )
{
RemoteArguments . Add ( String . Format ( "-remoteini={0}" , GetRemotePath ( RemoteIniPath ) ) ) ;
}
2019-01-14 12:11:24 -05:00
2019-04-11 06:08:23 -04:00
if ( TargetDesc . ProjectFile ! = null )
2019-01-14 12:11:24 -05:00
{
RemoteArguments . Add ( String . Format ( "-Project={0}" , GetRemotePath ( TargetDesc . ProjectFile ) ) ) ;
}
2019-03-13 11:41:55 -04:00
foreach ( string LocalArgument in TargetDesc . AdditionalArguments )
2019-01-14 12:11:24 -05:00
{
int EqualsIdx = LocalArgument . IndexOf ( '=' ) ;
if ( EqualsIdx = = - 1 )
{
RemoteArguments . Add ( LocalArgument ) ;
continue ;
}
string Key = LocalArgument . Substring ( 0 , EqualsIdx ) ;
string Value = LocalArgument . Substring ( EqualsIdx + 1 ) ;
if ( Key . Equals ( "-Log" , StringComparison . InvariantCultureIgnoreCase ) )
{
// We are already writing to the local log file. The remote will produce a different log (RemoteLogFile)
continue ;
}
if ( Key . Equals ( "-Manifest" , StringComparison . InvariantCultureIgnoreCase ) & & LocalManifestFiles ! = null )
{
LocalManifestFiles . Add ( new FileReference ( Value ) ) ;
continue ;
}
string RemoteArgument = LocalArgument ;
foreach ( RemoteMapping Mapping in Mappings )
{
if ( Value . StartsWith ( Mapping . LocalDirectory . FullName , StringComparison . InvariantCultureIgnoreCase ) )
{
RemoteArgument = String . Format ( "{0}={1}" , Key , GetRemotePath ( Value ) ) ;
break ;
}
}
RemoteArguments . Add ( RemoteArgument ) ;
}
return RemoteArguments ;
}
2018-08-16 13:53:43 -04:00
/// <summary>
/// Runs the actool utility on a directory to create an Assets.car file
/// </summary>
/// <param name="Platform">The target platform</param>
/// <param name="InputDir">Input directory containing assets</param>
/// <param name="OutputFile">Path to the Assets.car file to produce</param>
2019-05-24 11:51:54 -04:00
public void RunAssetCatalogTool ( UnrealTargetPlatform Platform , DirectoryReference InputDir , FileReference OutputFile )
2018-08-16 13:53:43 -04:00
{
Log . TraceInformation ( "Running asset catalog tool for {0}: {1} -> {2}" , Platform , InputDir , OutputFile ) ;
string RemoteInputDir = GetRemotePath ( InputDir ) ;
UploadDirectory ( InputDir ) ;
string RemoteOutputFile = GetRemotePath ( OutputFile ) ;
Execute ( RemoteBaseDir , String . Format ( "rm -f {0}" , EscapeShellArgument ( RemoteOutputFile ) ) ) ;
string RemoteOutputDir = Path . GetDirectoryName ( RemoteOutputFile ) . Replace ( Path . DirectorySeparatorChar , '/' ) ;
Execute ( RemoteBaseDir , String . Format ( "mkdir -p {0}" , EscapeShellArgument ( RemoteOutputDir ) ) ) ;
string RemoteArguments = IOSToolChain . GetAssetCatalogArgs ( Platform , RemoteInputDir , RemoteOutputDir ) ;
if ( Execute ( RemoteBaseDir , String . Format ( "/usr/bin/xcrun {0}" , RemoteArguments ) ) ! = 0 )
{
throw new BuildException ( "Failed to run actool." ) ;
}
DownloadFile ( OutputFile ) ;
}
/// <summary>
/// Convers a remote path into local form
/// </summary>
/// <param name="RemotePath">The remote filename</param>
/// <returns>Local filename corresponding to the remote path</returns>
private FileReference GetLocalPath ( string RemotePath )
{
foreach ( RemoteMapping Mapping in Mappings )
{
if ( RemotePath . StartsWith ( Mapping . RemoteDirectory , StringComparison . InvariantCultureIgnoreCase ) & & RemotePath . Length > Mapping . RemoteDirectory . Length & & RemotePath [ Mapping . RemoteDirectory . Length ] = = '/' )
{
return FileReference . Combine ( Mapping . LocalDirectory , RemotePath . Substring ( Mapping . RemoteDirectory . Length + 1 ) ) ;
}
}
throw new BuildException ( "Unable to map remote path '{0}' to local path" , RemotePath ) ;
}
/// <summary>
/// Converts a local path into a remote one
/// </summary>
/// <param name="LocalPath">The local path to convert</param>
/// <returns>Equivalent remote path</returns>
private string GetRemotePath ( FileSystemReference LocalPath )
{
return GetRemotePath ( LocalPath . FullName ) ;
}
/// <summary>
/// Converts a local path into a remote one
/// </summary>
/// <param name="LocalPath">The local path to convert</param>
/// <returns>Equivalent remote path</returns>
private string GetRemotePath ( string LocalPath )
{
return String . Format ( "{0}/{1}" , RemoteBaseDir , LocalPath . Replace ( ":" , "" ) . Replace ( "\\" , "/" ) . Replace ( " " , "_" ) ) ;
}
/// <summary>
/// Gets the local path in Cygwin format (eg. /cygdrive/C/...)
/// </summary>
/// <param name="InPath">Local path</param>
/// <returns>Path in cygwin format</returns>
private static string GetLocalCygwinPath ( FileSystemReference InPath )
{
if ( InPath . FullName . Length < 2 | | InPath . FullName [ 1 ] ! = ':' )
{
throw new BuildException ( "Invalid local path for converting to cygwin format ({0})." , InPath ) ;
}
return String . Format ( "/cygdrive/{0}{1}" , InPath . FullName . Substring ( 0 , 1 ) , InPath . FullName . Substring ( 2 ) . Replace ( '\\' , '/' ) ) ;
}
/// <summary>
/// Escapes spaces in a shell command argument
/// </summary>
/// <param name="Argument">The argument to escape</param>
/// <returns>The escaped argument</returns>
private static string EscapeShellArgument ( string Argument )
{
return Argument . Replace ( " " , "\\ " ) ;
}
/// <summary>
/// Upload a single file to the remote
/// </summary>
/// <param name="LocalFile">The file to upload</param>
void UploadFile ( FileReference LocalFile )
{
string RemoteFile = GetRemotePath ( LocalFile ) ;
string RemoteDirectory = GetRemotePath ( LocalFile . Directory ) ;
List < string > Arguments = new List < string > ( CommonRsyncArguments ) ;
Arguments . Add ( String . Format ( "--rsync-path=\"mkdir -p {0} && rsync\"" , RemoteDirectory ) ) ;
Arguments . Add ( String . Format ( "\"{0}\"" , GetLocalCygwinPath ( LocalFile ) ) ) ;
2018-10-30 10:24:29 -04:00
Arguments . Add ( String . Format ( "\"{0}@{1}\":'{2}'" , UserName , ServerName , RemoteFile ) ) ;
2018-08-16 13:53:43 -04:00
Arguments . Add ( "-q" ) ;
int Result = Rsync ( String . Join ( " " , Arguments ) ) ;
if ( Result ! = 0 )
{
throw new BuildException ( "Error while running Rsync (exit code {0})" , Result ) ;
}
}
2018-10-04 21:12:36 -04:00
/// <summary>
/// Upload a single file to the remote
/// </summary>
/// <param name="LocalDirectory">The base directory to copy</param>
/// <param name="RemoteDirectory">The remote directory</param>
/// <param name="LocalFileList">The file to upload</param>
void UploadFiles ( DirectoryReference LocalDirectory , string RemoteDirectory , FileReference LocalFileList )
{
List < string > Arguments = new List < string > ( BasicRsyncArguments ) ;
2018-10-11 15:03:02 -04:00
Arguments . Add ( String . Format ( "--rsync-path=\"mkdir -p {0} && rsync\"" , RemoteDirectory ) ) ;
2018-10-04 21:12:36 -04:00
Arguments . Add ( String . Format ( "--files-from=\"{0}\"" , GetLocalCygwinPath ( LocalFileList ) ) ) ;
Arguments . Add ( String . Format ( "\"{0}/\"" , GetLocalCygwinPath ( LocalDirectory ) ) ) ;
Arguments . Add ( String . Format ( "\"{0}@{1}\":'{2}/'" , UserName , ServerName , RemoteDirectory ) ) ;
Arguments . Add ( "-q" ) ;
int Result = Rsync ( String . Join ( " " , Arguments ) ) ;
if ( Result ! = 0 )
{
throw new BuildException ( "Error while running Rsync (exit code {0})" , Result ) ;
}
}
2018-08-16 13:53:43 -04:00
/// <summary>
/// Upload a single directory to the remote
/// </summary>
/// <param name="LocalDirectory">The local directory to upload</param>
void UploadDirectory ( DirectoryReference LocalDirectory )
{
string RemoteDirectory = GetRemotePath ( LocalDirectory ) ;
List < string > Arguments = new List < string > ( CommonRsyncArguments ) ;
Arguments . Add ( String . Format ( "--rsync-path=\"mkdir -p {0} && rsync\"" , RemoteDirectory ) ) ;
Arguments . Add ( String . Format ( "\"{0}/\"" , GetLocalCygwinPath ( LocalDirectory ) ) ) ;
Arguments . Add ( String . Format ( "\"{0}@{1}\":'{2}/'" , UserName , ServerName , RemoteDirectory ) ) ;
Arguments . Add ( "-q" ) ;
int Result = Rsync ( String . Join ( " " , Arguments ) ) ;
if ( Result ! = 0 )
{
throw new BuildException ( "Error while running Rsync (exit code {0})" , Result ) ;
}
}
/// <summary>
/// Uploads a directory to the remote using a specific filter list
/// </summary>
/// <param name="LocalDirectory">The local directory to copy from</param>
/// <param name="RemoteDirectory">The remote directory to copy to</param>
/// <param name="FilterLocations">List of paths to filter</param>
void UploadDirectory ( DirectoryReference LocalDirectory , string RemoteDirectory , List < FileReference > FilterLocations )
{
List < string > Arguments = new List < string > ( CommonRsyncArguments ) ;
Arguments . Add ( String . Format ( "--rsync-path=\"mkdir -p {0} && rsync\"" , RemoteDirectory ) ) ;
foreach ( FileReference FilterLocation in FilterLocations )
{
Arguments . Add ( String . Format ( "--filter=\"merge {0}\"" , GetLocalCygwinPath ( FilterLocation ) ) ) ;
}
Arguments . Add ( "--exclude='*'" ) ;
Arguments . Add ( String . Format ( "\"{0}/\"" , GetLocalCygwinPath ( LocalDirectory ) ) ) ;
Arguments . Add ( String . Format ( "\"{0}@{1}\":'{2}/'" , UserName , ServerName , RemoteDirectory ) ) ;
int Result = Rsync ( String . Join ( " " , Arguments ) ) ;
if ( Result ! = 0 )
{
throw new BuildException ( "Error while running Rsync (exit code {0})" , Result ) ;
}
}
/// <summary>
/// Upload all the files in the workspace for the current project
/// </summary>
void UploadWorkspace ( DirectoryReference TempDir )
{
// Path to the scripts to be uploaded
2021-06-11 18:20:44 -04:00
FileReference ScriptPathsFileName = FileReference . Combine ( Unreal . EngineDirectory , "Build" , "Rsync" , "RsyncEngineScripts.txt" ) ;
2018-08-16 13:53:43 -04:00
// Read the list of scripts to be uploaded
List < string > ScriptPaths = new List < string > ( ) ;
foreach ( string Line in FileReference . ReadAllLines ( ScriptPathsFileName ) )
{
string FileToUpload = Line . Trim ( ) ;
2018-10-04 21:12:36 -04:00
if ( FileToUpload . Length > 0 & & FileToUpload [ 0 ] ! = '#' )
2018-08-16 13:53:43 -04:00
{
ScriptPaths . Add ( FileToUpload ) ;
}
}
// Fixup the line endings
List < FileReference > TargetFiles = new List < FileReference > ( ) ;
foreach ( string ScriptPath in ScriptPaths )
{
2021-06-11 18:20:44 -04:00
FileReference SourceFile = FileReference . Combine ( Unreal . EngineDirectory , ScriptPath . TrimStart ( '/' ) ) ;
2018-08-16 13:53:43 -04:00
if ( ! FileReference . Exists ( SourceFile ) )
{
throw new BuildException ( "Missing script required for remote upload: {0}" , SourceFile ) ;
}
2021-06-11 18:20:44 -04:00
FileReference TargetFile = FileReference . Combine ( TempDir , SourceFile . MakeRelativeTo ( Unreal . EngineDirectory ) ) ;
2018-08-16 13:53:43 -04:00
if ( ! FileReference . Exists ( TargetFile ) | | FileReference . GetLastWriteTimeUtc ( TargetFile ) < FileReference . GetLastWriteTimeUtc ( SourceFile ) )
{
DirectoryReference . CreateDirectory ( TargetFile . Directory ) ;
string ScriptText = FileReference . ReadAllText ( SourceFile ) ;
FileReference . WriteAllText ( TargetFile , ScriptText . Replace ( "\r\n" , "\n" ) ) ;
}
TargetFiles . Add ( TargetFile ) ;
}
// Write a file that protects all the scripts from being overridden by the standard engine filters
FileReference ScriptProtectList = FileReference . Combine ( TempDir , "RsyncEngineScripts-Protect.txt" ) ;
using ( StreamWriter Writer = new StreamWriter ( ScriptProtectList . FullName ) )
{
foreach ( string ScriptPath in ScriptPaths )
{
Writer . WriteLine ( "protect {0}" , ScriptPath ) ;
}
}
// Upload these files to the remote
2018-10-04 21:12:36 -04:00
Log . TraceInformation ( "[Remote] Uploading scripts..." ) ;
2021-06-11 18:20:44 -04:00
UploadFiles ( TempDir , GetRemotePath ( Unreal . EngineDirectory ) , ScriptPathsFileName ) ;
2018-08-16 13:53:43 -04:00
2018-10-15 11:11:43 -04:00
// Upload the config files
Log . TraceInformation ( "[Remote] Uploading config files..." ) ;
UploadFile ( XmlConfig . CacheFile ) ;
2018-08-16 13:53:43 -04:00
// Upload the engine files
List < FileReference > EngineFilters = new List < FileReference > ( ) ;
EngineFilters . Add ( ScriptProtectList ) ;
2021-06-11 18:20:44 -04:00
if ( Unreal . IsEngineInstalled ( ) )
2018-10-16 14:21:36 -04:00
{
2021-06-11 18:20:44 -04:00
EngineFilters . Add ( FileReference . Combine ( Unreal . EngineDirectory , "Build" , "Rsync" , "RsyncEngineInstalled.txt" ) ) ;
2018-10-16 14:21:36 -04:00
}
2021-06-11 18:20:44 -04:00
EngineFilters . Add ( FileReference . Combine ( Unreal . EngineDirectory , "Build" , "Rsync" , "RsyncEngine.txt" ) ) ;
2018-10-04 21:12:36 -04:00
Log . TraceInformation ( "[Remote] Uploading engine files..." ) ;
2021-06-11 18:20:44 -04:00
UploadDirectory ( Unreal . EngineDirectory , GetRemotePath ( Unreal . EngineDirectory ) , EngineFilters ) ;
2018-08-16 13:53:43 -04:00
// Upload the project files
2019-04-11 06:08:23 -04:00
DirectoryReference ProjectDir = null ;
2021-06-11 18:20:44 -04:00
if ( ProjectFile ! = null & & ! ProjectFile . IsUnderDirectory ( Unreal . EngineDirectory ) )
2019-04-11 06:08:23 -04:00
{
ProjectDir = ProjectFile . Directory ;
}
else if ( ! string . IsNullOrEmpty ( UnrealBuildTool . GetRemoteIniPath ( ) ) )
{
ProjectDir = new DirectoryReference ( UnrealBuildTool . GetRemoteIniPath ( ) ) ;
2021-06-11 18:20:44 -04:00
if ( ProjectDir . IsUnderDirectory ( Unreal . EngineDirectory ) )
2019-04-11 06:08:23 -04:00
{
ProjectDir = null ;
}
}
if ( ProjectDir ! = null )
2018-08-16 13:53:43 -04:00
{
List < FileReference > ProjectFilters = new List < FileReference > ( ) ;
2020-01-31 10:34:10 -05:00
FileReference CustomFilter = FileReference . Combine ( ProjectDir , "Build" , "Rsync" , "RsyncProject.txt" ) ;
if ( FileReference . Exists ( CustomFilter ) )
2018-08-16 13:53:43 -04:00
{
2020-01-31 10:34:10 -05:00
ProjectFilters . Add ( CustomFilter ) ;
2018-08-16 13:53:43 -04:00
}
2021-06-11 18:20:44 -04:00
ProjectFilters . Add ( FileReference . Combine ( Unreal . EngineDirectory , "Build" , "Rsync" , "RsyncProject.txt" ) ) ;
2018-08-16 13:53:43 -04:00
2018-10-04 21:12:36 -04:00
Log . TraceInformation ( "[Remote] Uploading project files..." ) ;
2019-04-11 06:08:23 -04:00
UploadDirectory ( ProjectDir , GetRemotePath ( ProjectDir ) , ProjectFilters ) ;
2018-08-16 13:53:43 -04:00
}
2019-01-14 12:11:24 -05:00
2020-01-31 10:34:10 -05:00
if ( AdditionalPaths ! = null )
{
foreach ( DirectoryReference AdditionalPath in AdditionalPaths )
{
List < FileReference > CustomFilters = new List < FileReference > ( ) ;
FileReference CustomFilter = FileReference . Combine ( AdditionalPath , "Build" , "Rsync" , "RsyncProject.txt" ) ;
if ( FileReference . Exists ( CustomFilter ) )
{
CustomFilters . Add ( CustomFilter ) ;
}
2021-06-11 18:20:44 -04:00
CustomFilters . Add ( FileReference . Combine ( Unreal . EngineDirectory , "Build" , "Rsync" , "RsyncProject.txt" ) ) ;
2020-01-31 10:34:10 -05:00
Log . TraceInformation ( string . Format ( "[Remote] Uploading additional path files [{0}]..." , AdditionalPath . FullName ) ) ;
UploadDirectory ( AdditionalPath , GetRemotePath ( AdditionalPath ) , CustomFilters ) ;
}
}
2021-06-11 18:20:44 -04:00
Execute ( "/" , String . Format ( "rm -rf {0}/Intermediate/IOS/*.plist" , GetRemotePath ( Unreal . EngineDirectory ) ) ) ;
Execute ( "/" , String . Format ( "rm -rf {0}/Intermediate/TVOS/*.plist" , GetRemotePath ( Unreal . EngineDirectory ) ) ) ;
2019-06-04 15:45:23 -04:00
if ( ProjectFile ! = null )
{
Execute ( "/" , String . Format ( "rm -rf {0}/Intermediate/IOS/*.plist" , GetRemotePath ( ProjectFile . Directory ) ) ) ;
Execute ( "/" , String . Format ( "rm -rf {0}/Intermediate/TVOS/*.plist" , GetRemotePath ( ProjectFile . Directory ) ) ) ;
}
2019-05-15 08:03:24 -04:00
2021-01-11 21:17:03 -04:00
// Convert CRLF to LF for all shell scripts
2021-06-11 18:20:44 -04:00
Execute ( RemoteBaseDir , String . Format ( "for i in {0}/Build/BatchFiles/Mac/*.sh; do mv $i $i.crlf; tr -d '\r' < $i.crlf > $i; done" , EscapeShellArgument ( GetRemotePath ( Unreal . EngineDirectory ) ) ) ) ;
2021-01-11 21:17:03 -04:00
2019-01-14 12:11:24 -05:00
// Fixup permissions on any shell scripts
2021-06-11 18:20:44 -04:00
Execute ( RemoteBaseDir , String . Format ( "chmod +x {0}/Build/BatchFiles/Mac/*.sh" , EscapeShellArgument ( GetRemotePath ( Unreal . EngineDirectory ) ) ) ) ;
2018-08-16 13:53:43 -04:00
}
/// <summary>
/// Downloads a single file from the remote
/// </summary>
/// <param name="LocalFile">The file to download</param>
void DownloadFile ( FileReference LocalFile )
{
RemoteMapping Mapping = Mappings . FirstOrDefault ( x = > LocalFile . IsUnderDirectory ( x . LocalDirectory ) ) ;
if ( Mapping = = null )
{
throw new BuildException ( "File for download '{0}' is not under any mapped directory." , LocalFile ) ;
}
List < string > Arguments = new List < string > ( CommonRsyncArguments ) ;
Arguments . Add ( String . Format ( "\"{0}@{1}\":'{2}/{3}'" , UserName , ServerName , Mapping . RemoteDirectory , LocalFile . MakeRelativeTo ( Mapping . LocalDirectory ) . Replace ( '\\' , '/' ) ) ) ;
Arguments . Add ( String . Format ( "\"{0}/\"" , GetLocalCygwinPath ( LocalFile . Directory ) ) ) ;
Arguments . Add ( "-q" ) ;
int Result = Rsync ( String . Join ( " " , Arguments ) ) ;
if ( Result ! = 0 )
{
throw new BuildException ( "Unable to download '{0}' from the remote Mac (exit code {1})." , LocalFile , Result ) ;
}
}
/// <summary>
/// Download multiple files from the remote Mac
/// </summary>
/// <param name="Files">List of local files to download</param>
void DownloadFiles ( IEnumerable < FileReference > Files )
{
List < FileReference > [ ] FileGroups = new List < FileReference > [ Mappings . Count ] ;
for ( int Idx = 0 ; Idx < Mappings . Count ; Idx + + )
{
FileGroups [ Idx ] = new List < FileReference > ( ) ;
}
foreach ( FileReference File in Files )
{
int MappingIdx = Mappings . FindIndex ( x = > File . IsUnderDirectory ( x . LocalDirectory ) ) ;
if ( MappingIdx = = - 1 )
{
throw new BuildException ( "File for download '{0}' is not under the engine or project directory." , File ) ;
}
FileGroups [ MappingIdx ] . Add ( File ) ;
}
for ( int Idx = 0 ; Idx < Mappings . Count ; Idx + + )
{
if ( FileGroups [ Idx ] . Count > 0 )
{
2021-06-11 18:20:44 -04:00
FileReference DownloadListLocation = FileReference . Combine ( Unreal . EngineDirectory , "Intermediate" , "Rsync" , "Download.txt" ) ;
2018-08-16 13:53:43 -04:00
DirectoryReference . CreateDirectory ( DownloadListLocation . Directory ) ;
FileReference . WriteAllLines ( DownloadListLocation , FileGroups [ Idx ] . Select ( x = > x . MakeRelativeTo ( Mappings [ Idx ] . LocalDirectory ) . Replace ( '\\' , '/' ) ) ) ;
List < string > Arguments = new List < string > ( CommonRsyncArguments ) ;
Arguments . Add ( String . Format ( "--files-from=\"{0}\"" , GetLocalCygwinPath ( DownloadListLocation ) ) ) ;
Arguments . Add ( String . Format ( "\"{0}@{1}\":'{2}/'" , UserName , ServerName , Mappings [ Idx ] . RemoteDirectory ) ) ;
Arguments . Add ( String . Format ( "\"{0}/\"" , GetLocalCygwinPath ( Mappings [ Idx ] . LocalDirectory ) ) ) ;
int Result = Rsync ( String . Join ( " " , Arguments ) ) ;
if ( Result ! = 0 )
{
throw new BuildException ( "Unable to download files from remote Mac (exit code {0})" , Result ) ;
}
}
}
}
2019-04-14 09:07:30 -04:00
/// <summary>
/// Checks whether a directory exists on the remote machine
/// </summary>
/// <param name="LocalDirectory">Path to the directory on the local machine</param>
/// <returns>True if the remote directory exists</returns>
private bool RemoteDirectoryExists ( DirectoryReference LocalDirectory )
{
string RemoteDirectory = GetRemotePath ( LocalDirectory ) ;
2021-06-11 18:20:44 -04:00
return Execute ( Unreal . RootDirectory , String . Format ( "[ -d {0} ]" , EscapeShellArgument ( RemoteDirectory ) ) ) = = 0 ;
2019-04-14 09:07:30 -04:00
}
2019-02-12 09:39:55 -05:00
/// <summary>
/// Download a directory from the remote Mac
/// </summary>
/// <param name="LocalDirectory">Directory to download</param>
private void DownloadDirectory ( DirectoryReference LocalDirectory )
{
DirectoryReference . CreateDirectory ( LocalDirectory ) ;
string RemoteDirectory = GetRemotePath ( LocalDirectory ) ;
List < string > Arguments = new List < string > ( CommonRsyncArguments ) ;
Arguments . Add ( String . Format ( "\"{0}@{1}\":'{2}/'" , UserName , ServerName , RemoteDirectory ) ) ;
Arguments . Add ( String . Format ( "\"{0}/\"" , GetLocalCygwinPath ( LocalDirectory ) ) ) ;
int Result = Rsync ( String . Join ( " " , Arguments ) ) ;
if ( Result ! = 0 )
{
throw new BuildException ( "Unable to download '{0}' from the remote Mac (exit code {1})." , LocalDirectory , Result ) ;
}
}
2018-08-16 13:53:43 -04:00
/// <summary>
/// Execute Rsync
/// </summary>
/// <param name="Arguments">Arguments for the Rsync command</param>
/// <returns>Exit code from Rsync</returns>
private int Rsync ( string Arguments )
{
using ( Process RsyncProcess = new Process ( ) )
{
2021-04-20 10:44:53 -04:00
DataReceivedEventHandler OutputHandler = ( E , Args ) = > { RsyncOutput ( Args , false ) ; } ;
DataReceivedEventHandler ErrorHandler = ( E , Args ) = > { RsyncOutput ( Args , true ) ; } ;
2018-08-16 13:53:43 -04:00
RsyncProcess . StartInfo . FileName = RsyncExe . FullName ;
RsyncProcess . StartInfo . Arguments = Arguments ;
RsyncProcess . StartInfo . WorkingDirectory = SshExe . Directory . FullName ;
2021-04-20 10:44:53 -04:00
RsyncProcess . OutputDataReceived + = OutputHandler ;
RsyncProcess . ErrorDataReceived + = ErrorHandler ;
2018-08-16 13:53:43 -04:00
Log . TraceLog ( "[Rsync] {0} {1}" , Utils . MakePathSafeToUseWithCommandLine ( RsyncProcess . StartInfo . FileName ) , RsyncProcess . StartInfo . Arguments ) ;
return Utils . RunLocalProcess ( RsyncProcess ) ;
}
}
/// <summary>
/// Handles data output by rsync
/// </summary>
/// <param name="Args">The received data</param>e
2021-04-20 10:44:53 -04:00
/// <param name="bStdErr">whether the data was received on stderr</param>
private void RsyncOutput ( DataReceivedEventArgs Args , bool bStdErr )
2018-08-16 13:53:43 -04:00
{
2021-05-27 13:40:37 -04:00
if ( Args . Data ! = null )
2018-08-16 13:53:43 -04:00
{
2021-04-20 10:44:53 -04:00
if ( bStdErr )
{
Log . TraceError ( " {0}" , Args . Data ) ;
}
else
{
Log . TraceInformation ( " {0}" , Args . Data ) ;
}
2018-08-16 13:53:43 -04:00
}
}
/// <summary>
/// Execute a command on the remote in the remote equivalent of a local directory
/// </summary>
/// <param name="WorkingDir"></param>
/// <param name="Command"></param>
/// <returns></returns>
public int Execute ( DirectoryReference WorkingDir , string Command )
{
return Execute ( GetRemotePath ( WorkingDir ) , Command ) ;
}
/// <summary>
/// Execute a remote command, capturing the output text
/// </summary>
/// <param name="WorkingDirectory">The remote working directory</param>
/// <param name="Command">Command to be executed</param>
/// <returns></returns>
protected int Execute ( string WorkingDirectory , string Command )
{
string FullCommand = String . Format ( "cd {0} && {1}" , EscapeShellArgument ( WorkingDirectory ) , Command ) ;
2021-05-27 13:40:37 -04:00
using ( Process SSHProcess = new Process ( ) )
2018-08-16 13:53:43 -04:00
{
2021-04-20 10:44:53 -04:00
DataReceivedEventHandler OutputHandler = ( E , Args ) = > { SshOutput ( Args , false ) ; } ;
DataReceivedEventHandler ErrorHandler = ( E , Args ) = > { SshOutput ( Args , true ) ; } ;
2018-08-16 13:53:43 -04:00
SSHProcess . StartInfo . FileName = SshExe . FullName ;
SSHProcess . StartInfo . WorkingDirectory = SshExe . Directory . FullName ;
2021-04-20 10:44:53 -04:00
SSHProcess . StartInfo . Arguments = String . Format ( "{0} {1}" , String . Join ( " " , CommonSshArguments ) , FullCommand ) ;
2018-08-16 13:53:43 -04:00
SSHProcess . OutputDataReceived + = OutputHandler ;
2021-04-20 10:44:53 -04:00
SSHProcess . ErrorDataReceived + = ErrorHandler ;
2018-08-16 13:53:43 -04:00
Log . TraceLog ( "[SSH] {0} {1}" , Utils . MakePathSafeToUseWithCommandLine ( SSHProcess . StartInfo . FileName ) , SSHProcess . StartInfo . Arguments ) ;
return Utils . RunLocalProcess ( SSHProcess ) ;
}
}
/// <summary>
/// Handler for output from running remote SSH commands
/// </summary>
/// <param name="Args"></param>
2021-04-20 10:44:53 -04:00
/// <param name="bStdErr">whether the data was received on stderr</param>
private void SshOutput ( DataReceivedEventArgs Args , bool bStdErr )
2018-08-16 13:53:43 -04:00
{
2021-05-27 13:40:37 -04:00
if ( Args . Data ! = null )
2018-08-16 13:53:43 -04:00
{
string FormattedOutput = ConvertRemotePathsToLocal ( Args . Data ) ;
2021-04-20 10:44:53 -04:00
if ( bStdErr )
{
Log . TraceError ( " {0}" , FormattedOutput ) ;
}
else
{
Log . TraceInformation ( " {0}" , FormattedOutput ) ;
}
2018-08-16 13:53:43 -04:00
}
}
2018-10-03 09:21:19 -04:00
/// <summary>
/// Execute a remote command, capturing the output text
/// </summary>
/// <param name="Command">Command to be executed</param>
/// <param name="Output">Receives the output text</param>
/// <returns></returns>
protected int ExecuteAndCaptureOutput ( string Command , out StringBuilder Output )
{
StringBuilder FullCommand = new StringBuilder ( ) ;
foreach ( string CommonSshArgument in CommonSshArguments )
{
FullCommand . AppendFormat ( "{0} " , CommonSshArgument ) ;
}
FullCommand . Append ( Command . Replace ( "\"" , "\\\"" ) ) ;
using ( Process SSHProcess = new Process ( ) )
{
Output = new StringBuilder ( ) ;
StringBuilder OutputLocal = Output ;
DataReceivedEventHandler OutputHandler = ( E , Args ) = > { if ( Args . Data ! = null ) { OutputLocal . Append ( Args . Data ) ; } } ;
SSHProcess . StartInfo . FileName = SshExe . FullName ;
SSHProcess . StartInfo . WorkingDirectory = SshExe . Directory . FullName ;
SSHProcess . StartInfo . Arguments = FullCommand . ToString ( ) ;
SSHProcess . OutputDataReceived + = OutputHandler ;
SSHProcess . ErrorDataReceived + = OutputHandler ;
Log . TraceLog ( "[SSH] {0} {1}" , Utils . MakePathSafeToUseWithCommandLine ( SSHProcess . StartInfo . FileName ) , SSHProcess . StartInfo . Arguments ) ;
return Utils . RunLocalProcess ( SSHProcess ) ;
}
}
2018-08-16 13:53:43 -04:00
/// <summary>
/// Converts any remote paths within the given string to local format
/// </summary>
/// <param name="Text">The text containing strings to convert</param>
/// <returns>The string with paths converted to local format</returns>
private string ConvertRemotePathsToLocal ( string Text )
{
// Try to match any source file with the remote base directory in front of it
string Pattern = String . Format ( "(?<![a-zA-Z=]){0}[^:]*\\.(?:cpp|inl|h|hpp|hh|txt)(?![a-zA-Z])" , Regex . Escape ( RemoteBaseDir ) ) ;
// Find the matches, and early out if there are none
MatchCollection Matches = Regex . Matches ( Text , Pattern , RegexOptions . IgnoreCase ) ;
if ( Matches . Count = = 0 )
{
return Text ;
}
// Replace any remote paths with local ones
StringBuilder Result = new StringBuilder ( ) ;
int StartIdx = 0 ;
foreach ( Match Match in Matches )
{
// Append the text leading up to this path
Result . Append ( Text , StartIdx , Match . Index - StartIdx ) ;
// Try to convert the path
string Path = Match . Value ;
foreach ( RemoteMapping Mapping in Mappings )
{
if ( Path . StartsWith ( Mapping . RemoteDirectory ) )
{
Path = Mapping . LocalDirectory + Path . Substring ( Mapping . RemoteDirectory . Length ) . Replace ( '/' , '\\' ) ;
break ;
}
}
// Append the path to the output string
Result . Append ( Path ) ;
// Move past this match
StartIdx = Match . Index + Match . Length ;
}
Result . Append ( Text , StartIdx , Text . Length - StartIdx ) ;
return Result . ToString ( ) ;
}
}
}