// Copyright 1998-2014 Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.IO; using AutomationTool; using System.Runtime.Serialization; using System.Net; using System.Reflection; using System.Text.RegularExpressions; using UnrealBuildTool; namespace EpicGames.MCP.Automation { using EpicGames.MCP.Config; /// /// Utility class to provide commit/rollback functionality via an RAII-like functionality. /// Usage is to provide a rollback action that will be called on Dispose if the Commit() method is not called. /// This is expected to be used from within a using() ... clause. /// public class CommitRollbackTransaction : IDisposable { /// /// Track whether the transaction will be committed. /// private bool IsCommitted = false; /// /// /// private System.Action RollbackAction; /// /// Call when you want to commit your transaction. Ensures the Rollback action is not called on Dispose(). /// public void Commit() { IsCommitted = true; } /// /// Constructor /// /// Action to be executed to rollback the transaction. public CommitRollbackTransaction(System.Action InRollbackAction) { RollbackAction = InRollbackAction; } /// /// Rollback the transaction if its not committed on Dispose. /// public void Dispose() { if (!IsCommitted) { RollbackAction(); } } } /// /// Enum that defines the MCP backend-compatible platform /// public enum MCPPlatform { /// /// MCP doesn't care about Win32 vs. Win64 /// Windows, /// /// Only other platform MCP understands is Mac. /// Mac, } /// /// Enum that defines CDN types /// public enum CDNType { /// /// Internal HTTP CDN server /// Internal, /// /// Production HTTP CDN server /// Production, } /// /// Class that holds common state used to control the BuildPatchTool build commands that chunk and create patching manifests and publish build info to the BuildInfoService. /// public class BuildPatchToolStagingInfo { /// /// The currently running command, used to get command line overrides /// public BuildCommand OwnerCommand; /// /// name of the app. Can't always use this to define the staging dir because some apps are not staged to a place that matches their AppName. /// public readonly string AppName; /// /// Usually the base name of the app. Used to get the MCP key from a branch dictionary. /// public readonly string McpConfigKey; /// /// ID of the app (needed for the BuildPatchTool) /// public readonly int AppID; /// /// BuildVersion of the App we are staging. /// public readonly string BuildVersion; /// /// Directory where builds will be staged. Rooted at the BuildRootPath, using a subfolder passed in the ctor, /// and using BuildVersion/PlatformName to give each builds their own home. /// public readonly string StagingDir; /// /// Path to the CloudDir where chunks will be written (relative to the BuildRootPath) /// This is used to copy to the web server, so it can use the same relative path to the root build directory. /// This allows file to be either copied from the local file system or the webserver using the same relative paths. /// public readonly string CloudDirRelativePath; /// /// full path to the CloudDir where chunks and manifests should be staged. Rooted at the BuildRootPath, using a subfolder pass in the ctor. /// public readonly string CloudDir; /// /// Platform we are staging for. /// public readonly MCPPlatform Platform; /// /// Gets the base filename of the manifest that would be created by invoking the BuildPatchTool with the given parameters. /// public string ManifestFilename { get { var BaseFilename = AppName + BuildVersion + "-" + Platform.ToString() + ".manifest"; return Regex.Replace(BaseFilename, @"\s+", ""); // Strip out whitespace in order to be compatible with BuildPatchTool } } /// /// Determine the platform name (Win32/64 becomes Windows, Mac is Mac, the rest we don't currently understand) /// static public MCPPlatform ToMCPPlatform(UnrealTargetPlatform TargetPlatform) { if (TargetPlatform != UnrealTargetPlatform.Win64 && TargetPlatform != UnrealTargetPlatform.Win32 && TargetPlatform != UnrealTargetPlatform.Mac) { throw new AutomationException("Platform {0} is not properly supported by the MCP backend yet", TargetPlatform); } return (TargetPlatform == UnrealTargetPlatform.Win64 || TargetPlatform == UnrealTargetPlatform.Win32) ? MCPPlatform.Windows : MCPPlatform.Mac; } /// /// Determine the platform name (Win32/64 becomes Windows, Mac is Mac, the rest we don't currently understand) /// static public UnrealTargetPlatform FromMCPPlatform(MCPPlatform TargetPlatform) { if (TargetPlatform != MCPPlatform.Windows && TargetPlatform != MCPPlatform.Mac) { throw new AutomationException("Platform {0} is not properly supported by the MCP backend yet", TargetPlatform); } return (TargetPlatform == MCPPlatform.Windows) ? UnrealTargetPlatform.Win64 : UnrealTargetPlatform.Mac; } /// /// Returns the build root path (P:\Builds on build machines usually) /// /// static public string GetBuildRootPath() { return CommandUtils.P4Enabled && CommandUtils.AllowSubmit ? CommandUtils.RootSharedTempStorageDirectory() : CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds"); } /// /// Basic constructor. /// /// /// /// /// /// Relative path from the BuildRootPath where files will be staged. Commonly matches the AppName. public BuildPatchToolStagingInfo(BuildCommand InOwnerCommand, string InAppName, string InMcpConfigKey, int InAppID, string InBuildVersion, UnrealTargetPlatform platform, string stagingDirRelativePath) : this(InOwnerCommand, InAppName, InMcpConfigKey, InAppID, InBuildVersion, ToMCPPlatform(platform), stagingDirRelativePath) { } /// /// Basic constructor. /// /// /// /// /// /// Relative path from the BuildRootPath where files will be staged. Commonly matches the AppName. public BuildPatchToolStagingInfo(BuildCommand InOwnerCommand, string InAppName, string InMcpConfigKey, int InAppID, string InBuildVersion, MCPPlatform platform, string stagingDirRelativePath) { OwnerCommand = InOwnerCommand; AppName = InAppName; McpConfigKey = InMcpConfigKey; AppID = InAppID; BuildVersion = InBuildVersion; Platform = platform; var BuildRootPath = GetBuildRootPath(); StagingDir = CommandUtils.CombinePaths(BuildRootPath, stagingDirRelativePath, BuildVersion, Platform.ToString()); CloudDirRelativePath = CommandUtils.CombinePaths(stagingDirRelativePath, "CloudDir"); CloudDir = CommandUtils.CombinePaths(BuildRootPath, CloudDirRelativePath); } } /// /// Class that provides programmatic access to the BuildPatchTool /// public abstract class BuildPatchToolBase { /// /// Controls the chunking type used by the buildinfo server (nochunks parameter) /// public enum ChunkType { /// /// Chunk the files /// Chunk, /// /// Don't chunk the files, just build a file manifest. /// NoChunk, } public class PatchGenerationOptions { /// /// Staging information /// public BuildPatchToolStagingInfo StagingInfo; /// /// Matches the corresponding BuildPatchTool command line argument. /// public string BuildRoot; /// /// Matches the corresponding BuildPatchTool command line argument. /// public string FileIgnoreList; /// /// Matches the corresponding BuildPatchTool command line argument. /// public string AppLaunchCmd; /// /// Matches the corresponding BuildPatchTool command line argument. /// public string AppLaunchCmdArgs; /// /// Corresponds to the -nochunks parameter /// public ChunkType AppChunkType; /// /// Matches the corresponding BuildPatchTool command line argument. /// public MCPPlatform Platform; } public class CompactifyOptions { /// /// If specified, BuildPatchTool will run a compactify on this directory. /// public string CompactifyDirectory; /// /// Corresponds to the -preview parameter /// public bool bPreviewCompactify; } static BuildPatchToolBase Handler = null; public static BuildPatchToolBase Get() { if (Handler == null) { Assembly[] LoadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); foreach (var Dll in LoadedAssemblies) { Type[] AllTypes = Dll.GetTypes(); foreach (var PotentialConfigType in AllTypes) { if (PotentialConfigType != typeof(BuildPatchToolBase) && typeof(BuildPatchToolBase).IsAssignableFrom(PotentialConfigType)) { Handler = Activator.CreateInstance(PotentialConfigType) as BuildPatchToolBase; break; } } } if (Handler == null) { throw new AutomationException("Attempt to use BuildPatchToolBase.Get() and it doesn't appear that there are any modules that implement this class."); } } return Handler; } /// /// Runs the Build Patch Tool executable to generate patch data using the supplied parameters. /// /// Parameters which will be passed to the patch tool generation process public abstract void Execute(PatchGenerationOptions Opts); /// /// Runs the Build Patch Tool executable to compactify a cloud directory using the supplied parameters. /// /// Parameters which will be passed to the compactify process public abstract void Execute(CompactifyOptions Opts); } /// /// Helper class /// public abstract class BuildInfoPublisherBase { static BuildInfoPublisherBase Handler = null; public static BuildInfoPublisherBase Get() { if (Handler == null) { Assembly[] LoadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); foreach (var Dll in LoadedAssemblies) { Type[] AllTypes = Dll.GetTypes(); foreach (var PotentialConfigType in AllTypes) { if (PotentialConfigType != typeof(BuildInfoPublisherBase) && typeof(BuildInfoPublisherBase).IsAssignableFrom(PotentialConfigType)) { Handler = Activator.CreateInstance(PotentialConfigType) as BuildInfoPublisherBase; break; } } } if (Handler == null) { throw new AutomationException("Attempt to use BuildInfoPublisherBase.Get() and it doesn't appear that there are any modules that implement this class."); } } return Handler; } /// /// Given a MCPStagingInfo defining our build info, posts the build to the MCP BuildInfo Service. /// /// Staging Info describing the BuildInfo to post. abstract public void PostBuildInfo(BuildPatchToolStagingInfo stagingInfo); /// /// Given a MCPStagingInfo defining our build info and a MCP config name, posts the build to the requested MCP BuildInfo Service. /// /// Staging Info describing the BuildInfo to post. /// Name of which MCP config to post to. abstract public void PostBuildInfo(BuildPatchToolStagingInfo StagingInfo, string McpConfigName); /// /// Given a BuildVersion defining our a build, return the labels applied to that build /// /// Build version to return labels for. /// Which BuildInfo backend to get labels from for this promotion attempt. abstract public List GetBuildLabels(BuildPatchToolStagingInfo StagingInfo, string McpConfigName); /// /// Get a label string for the specific Platform requested. /// /// Base of label /// Platform to add to base label. abstract public string GetLabelWithPlatform(string DestinationLabel, MCPPlatform Platform); /// /// Get a BuildVersion string with the Platform concatenated on. /// /// Base of label /// Platform to add to base label. abstract public string GetBuildVersionWithPlatform(BuildPatchToolStagingInfo StagingInfo); /// /// /// /// Application name to check the label in /// Label name to get the build for /// Which BuildInfo backend to label the build in. /// abstract public string GetLabeledBuildVersion(string AppName, string LabelName, string McpConfigName); /// /// Apply the requested label to the requested build in the BuildInfo backend for the requested MCP environment /// /// Staging info for the build to label. /// Label, including platform, to apply /// Which BuildInfo backend to label the build in. abstract public void LabelBuild(BuildPatchToolStagingInfo StagingInfo, string DestinationLabelWithPlatform, string McpConfigName); /// /// Informs Patcher Service of a new build availability after async labeling is complete /// (this usually means the build was copied to a public file server before the label could be applied). /// /// Parent command /// Application name that the patcher service will use. /// BuildVersion string that the patcher service will use. /// Relative path to the Manifest file relative to the global build root (which is like P:\Builds) /// Name of the label that we will be setting. abstract public void BuildPromotionCompleted(BuildPatchToolStagingInfo stagingInfo, string AppName, string BuildVersion, string ManifestRelativePath, string PlatformName, string LabelName); /// /// Mounts the production CDN share (allows overriding via -CDNDrive command line arg) /// /// /// Path to the share (allows for override) abstract public string MountProductionCDNShare(); /// /// Mounts the production CDN share (allows overriding via -CDNDrive command line arg) /// /// /// Path to the share (allows for override) abstract public string MountInternalCDNShare(); /// /// Checks for a stagingInfo's manifest on the production CDN. /// /// Staging info used to determine where the chunks are to copy. abstract public bool IsManifestOnProductionCDN(BuildPatchToolStagingInfo stagingInfo); /// /// Copies chunks from a staged location to the production CDN. /// /// Build command (used to allow the -CDNDrive cmdline override). /// Staging info used to determine where the chunks are to copy. abstract public void CopyChunksToProductionCDN(BuildPatchToolStagingInfo stagingInfo); /// /// Copies chunks from a staged location to the production CDN. /// NOTE: This code assumes the location of the BuildRootPath at the time this build /// by calling (usually P:\Builds). /// If this path changes then this code posting older builds will break because we won't know /// where the BuildRootPath for the older build was! /// /// Build command (used to allow the -CDNDrive cmdline override). /// relative path to the manifest file from the build info service abstract public void CopyChunksToProductionCDN(string manifestUrlPath); } /// /// Helpers for using the MCP account service /// public abstract class McpAccountServiceBase { static McpAccountServiceBase Handler = null; public static McpAccountServiceBase Get() { if (Handler == null) { Assembly[] LoadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); foreach (var Dll in LoadedAssemblies) { Type[] AllTypes = Dll.GetTypes(); foreach (var PotentialConfigType in AllTypes) { if (PotentialConfigType != typeof(McpAccountServiceBase) && typeof(McpAccountServiceBase).IsAssignableFrom(PotentialConfigType)) { Handler = Activator.CreateInstance(PotentialConfigType) as McpAccountServiceBase; break; } } } if (Handler == null) { throw new AutomationException("Attempt to use McpAccountServiceBase.Get() and it doesn't appear that there are any modules that implement this class."); } } return Handler; } public abstract string GetClientToken(BuildPatchToolStagingInfo StagingInfo); public abstract string GetClientToken(McpConfigData McpConfig); public abstract string SendWebRequest(WebRequest Upload, string Method, string ContentType, byte[] Data); } } namespace EpicGames.MCP.Config { /// /// Class for retrieving MCP configuration data /// public class McpConfigHelper { // List of configs is cached off for fetching from multiple times private static Dictionary Configs; public static McpConfigData Find(string ConfigName) { if (Configs == null) { // Load all secret configs by trying to instantiate all classes derived from McpConfig from all loaded DLLs. // Note that we're using the default constructor on the secret config types. Configs = new Dictionary(); Assembly[] LoadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); foreach (var Dll in LoadedAssemblies) { Type[] AllTypes = Dll.GetTypes(); foreach (var PotentialConfigType in AllTypes) { if (PotentialConfigType != typeof(McpConfigData) && typeof(McpConfigData).IsAssignableFrom(PotentialConfigType)) { try { McpConfigData Config = Activator.CreateInstance(PotentialConfigType) as McpConfigData; if (Config != null) { Configs.Add(Config.Name, Config); } } catch { BuildCommand.LogWarning("Unable to create McpConfig: {0}", PotentialConfigType.Name); } } } } } McpConfigData LoadedConfig; Configs.TryGetValue(ConfigName, out LoadedConfig); if (LoadedConfig == null) { throw new AutomationException("Unable to find requested McpConfig: {0}", ConfigName); } return LoadedConfig; } } // Class for storing mcp configuration data public class McpConfigData { public McpConfigData(string InName, string InAccountBaseUrl, string InFortniteBaseUrl, string InBuildInfoBaseUrl, string InLauncherBaseUrl, string InBuildInfoV2BaseUrl, string InLauncherV2BaseUrl, string InClientId, string InClientSecret) { Name = InName; AccountBaseUrl = InAccountBaseUrl; FortniteBaseUrl = InFortniteBaseUrl; BuildInfoBaseUrl = InBuildInfoBaseUrl; LauncherBaseUrl = InLauncherBaseUrl; BuildInfoV2BaseUrl = InBuildInfoV2BaseUrl; LauncherV2BaseUrl = InLauncherV2BaseUrl; ClientId = InClientId; ClientSecret = InClientSecret; } public string Name; public string AccountBaseUrl; public string FortniteBaseUrl; public string BuildInfoBaseUrl; public string LauncherBaseUrl; public string BuildInfoV2BaseUrl; public string LauncherV2BaseUrl; public string ClientId; public string ClientSecret; public void SpewValues() { CommandUtils.Log("Name : {0}", Name); CommandUtils.Log("AccountBaseUrl : {0}", AccountBaseUrl); CommandUtils.Log("FortniteBaseUrl : {0}", FortniteBaseUrl); CommandUtils.Log("BuildInfoBaseUrl : {0}", BuildInfoBaseUrl); CommandUtils.Log("LauncherBaseUrl : {0}", LauncherBaseUrl); CommandUtils.Log("BuildInfoV2BaseUrl : {0}", BuildInfoV2BaseUrl); CommandUtils.Log("LauncherV2BaseUrl : {0}", LauncherV2BaseUrl); CommandUtils.Log("ClientId : {0}", ClientId); // we don't really want this in logs CommandUtils.Log("ClientSecret : {0}", ClientSecret); } /// /// Returns Base Urls of build info service(s). /// Will only return properties which have been populated. /// public IEnumerable BuildInfoBaseUrls { get { return new List { BuildInfoBaseUrl, BuildInfoV2BaseUrl }.Where(x => !string.IsNullOrEmpty(x)).Distinct(); } } /// /// Returns the build info base URL to use for get requests. /// Switching to version 2 can be achieved by overriding UseV2BuildInfoService in concrete subclasses and setting /// it to true. /// public string DefaultBuildInfoBaseUrl { get { return UseV2BuildInfoService ? BuildInfoV2BaseUrl : BuildInfoBaseUrl; } } protected virtual bool UseV2BuildInfoService { get { return false; } } } public class McpConfigMapper { static public McpConfigData FromMcpConfigKey(string McpConfigKey) { return McpConfigHelper.Find("MainGameDevNet"); } static public McpConfigData FromStagingInfo(EpicGames.MCP.Automation.BuildPatchToolStagingInfo StagingInfo) { string McpConfigNameToLookup = null; if (StagingInfo.OwnerCommand != null) { McpConfigNameToLookup = StagingInfo.OwnerCommand.ParseParamValue("MCPConfig"); } if (String.IsNullOrEmpty(McpConfigNameToLookup)) { return FromMcpConfigKey(StagingInfo.McpConfigKey); } return McpConfigHelper.Find(McpConfigNameToLookup); } } }