// Copyright Epic Games, Inc. All Rights Reserved.
using AutomationTool;
using EpicGames.BuildGraph;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
using EpicGames.Core;
using EpicGames.Serialization;
using UnrealBuildTool;
using UnrealBuildBase;
using Microsoft.Extensions.Logging;
using static AutomationTool.CommandUtils;
namespace AutomationTool.Tasks
{
enum SnapshotStorageType
{
Invalid,
Cloud,
Zen,
File,
}
///
/// Parameters for a task that exports an snapshot from ZenServer
///
public class ZenExportSnapshotTaskParameters
{
///
/// The project from which to export the snapshot
///
[TaskParameter(Optional = true)]
public FileReference Project;
///
/// The target platform(s) to export the snapshot for
///
[TaskParameter(Optional = true)]
public string Platform;
///
/// A file to create with information about the snapshot that was exported
///
[TaskParameter(Optional = true)]
public FileReference SnapshotDescriptorFile;
///
/// The type of destination to export the snapshot to (cloud, ...)
///
[TaskParameter]
public string DestinationStorageType;
///
/// The host name to use when exporting to a cloud destination
///
[TaskParameter(Optional = true)]
public string DestinationCloudHost;
///
/// The namespace to use when exporting to a cloud destination
///
[TaskParameter(Optional = true)]
public string DestinationCloudNamespace;
///
/// The identifier to use when exporting to a cloud destination
///
[TaskParameter(Optional = true)]
public string DestinationCloudIdentifier;
///
/// A custom bucket name to use when exporting to a cloud destination
///
[TaskParameter(Optional = true)]
public string DestinationCloudBucket;
}
///
/// Exports an snapshot from Zen to a specified destination.
///
[TaskElement("ZenExportSnapshot", typeof(ZenExportSnapshotTaskParameters))]
public class ZenExportSnapshotTask : BgTaskImpl
{
private class ExportSourceData
{
public bool IsLocalHost;
public string HostName;
public int HostPort;
public string ProjectId;
public string OplogId;
public string TargetPlatform;
}
///
/// Parameters for the task
///
ZenExportSnapshotTaskParameters Parameters;
///
/// Constructor.
///
/// Parameters for this task
public ZenExportSnapshotTask(ZenExportSnapshotTaskParameters InParameters)
{
Parameters = InParameters;
}
///
/// Execute the task.
///
/// Information about the current job
/// Set of build products produced by this node.
/// Mapping from tag names to the set of files they include
public override Task ExecuteAsync(JobContext Job, HashSet BuildProducts, Dictionary> TagNameToFileSet)
{
SnapshotStorageType DestinationStorageType = SnapshotStorageType.Invalid;
if (!string.IsNullOrEmpty(Parameters.DestinationStorageType))
{
DestinationStorageType = (SnapshotStorageType)Enum.Parse(typeof(SnapshotStorageType), Parameters.DestinationStorageType);
}
FileReference ProjectFile = Parameters.Project;
if(!FileReference.Exists(ProjectFile))
{
throw new AutomationException("Missing project file - {0}", ProjectFile.FullName);
}
List ExportSources = new List();
foreach (string Platform in Parameters.Platform.Split('+'))
{
DirectoryReference PlatformCookedDirectory = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "Cooked", Platform);
if (!DirectoryReference.Exists(PlatformCookedDirectory))
{
throw new AutomationException("Cook output directory not found ({0})", PlatformCookedDirectory.FullName);
}
FileReference ProjectStoreFile = FileReference.Combine(PlatformCookedDirectory, ".projectstore");
if (!FileReference.Exists(ProjectStoreFile))
{
continue;
}
byte[] ProjectStoreData = File.ReadAllBytes(ProjectStoreFile.FullName);
CbObject ProjectStoreObject = new CbField(ProjectStoreData).AsObject();
CbObject ZenServerObject = ProjectStoreObject["zenserver"].AsObject();
if (ZenServerObject != CbObject.Empty)
{
ExportSourceData NewExportSource = new ExportSourceData();
NewExportSource.IsLocalHost = ZenServerObject["islocalhost"].AsBool();
NewExportSource.HostName = ZenServerObject["hostname"].AsString("localhost");
NewExportSource.HostPort = ZenServerObject["hostport"].AsInt16(1337);
NewExportSource.ProjectId = ZenServerObject["projectid"].AsString();
NewExportSource.OplogId = ZenServerObject["oplogid"].AsString();
NewExportSource.TargetPlatform = Platform;
ExportSources.Add(NewExportSource);
}
}
// Get the executable path
FileReference ZenExe;
if(HostPlatform.Current.HostEditorPlatform == UnrealTargetPlatform.Win64)
{
ZenExe = ResolveFile("Engine/Binaries/Win64/zen.exe");
}
else
{
ZenExe = ResolveFile(String.Format("Engine/Binaries/{0}/zen", HostPlatform.Current.HostEditorPlatform.ToString()));
}
// Format the command lines
StringBuilder OplogSnapshotCommandline = new StringBuilder();
OplogSnapshotCommandline.AppendFormat("oplog-snapshot");
StringBuilder OplogExportCommandline = new StringBuilder();
OplogExportCommandline.AppendFormat("oplog-export");
switch (DestinationStorageType)
{
case SnapshotStorageType.Cloud:
if (string.IsNullOrEmpty(Parameters.DestinationCloudHost))
{
throw new AutomationException("Missing destination cloud host");
}
if (string.IsNullOrEmpty(Parameters.DestinationCloudNamespace))
{
throw new AutomationException("Missing destination cloud namespace");
}
if (string.IsNullOrEmpty(Parameters.DestinationCloudIdentifier))
{
throw new AutomationException("Missing destination cloud identifier");
}
string BucketName = Parameters.DestinationCloudBucket;
string ProjectNameAsBucketName = ProjectFile.GetFileNameWithoutAnyExtensions().ToLowerInvariant();
if (string.IsNullOrEmpty(BucketName))
{
BucketName = ProjectNameAsBucketName;
}
OplogExportCommandline.AppendFormat(" --cloud {0} --namespace {1} --bucket {2}", Parameters.DestinationCloudHost, Parameters.DestinationCloudNamespace, BucketName);
string[] ExportKeyIds = new string[ExportSources.Count];
int ExportIndex = 0;
foreach (ExportSourceData ExportSource in ExportSources)
{
String HostUrlArg = string.Format("--hosturl http://{0}:{1}", ExportSource.IsLocalHost ? "localhost" : ExportSource.HostName, ExportSource.HostPort);
StringBuilder SnapshotSingleSourceCommandline = new StringBuilder(OplogSnapshotCommandline.Length);
SnapshotSingleSourceCommandline.Append(OplogSnapshotCommandline);
SnapshotSingleSourceCommandline.AppendFormat(" {0} {1} {2}", HostUrlArg, ExportSource.ProjectId, ExportSource.OplogId);
Logger.LogInformation("Running '{Arg0} {Arg1}'", CommandUtils.MakePathSafeToUseWithCommandLine(ZenExe.FullName), SnapshotSingleSourceCommandline.ToString());
CommandUtils.RunAndLog(CommandUtils.CmdEnv, ZenExe.FullName, SnapshotSingleSourceCommandline.ToString(), Options: CommandUtils.ERunOptions.Default);
StringBuilder ExportSingleSourceCommandline = new StringBuilder(OplogExportCommandline.Length);
ExportSingleSourceCommandline.Append(OplogExportCommandline);
StringBuilder DestinationKeyBuilder = new StringBuilder();
DestinationKeyBuilder.AppendFormat("{0}.{1}.{2}", ProjectNameAsBucketName, Parameters.DestinationCloudIdentifier, ExportSource.OplogId);
ExportKeyIds[ExportIndex] = DestinationKeyBuilder.ToString().ToLowerInvariant();
IoHash DestinationKeyHash = IoHash.Compute(Encoding.UTF8.GetBytes(ExportKeyIds[ExportIndex]));
ExportSingleSourceCommandline.AppendFormat(" {0} --key {1} {2} {3}", HostUrlArg, DestinationKeyHash.ToString().ToLowerInvariant(), ExportSource.ProjectId, ExportSource.OplogId);
Logger.LogInformation("Running '{Arg0} {Arg1}'", CommandUtils.MakePathSafeToUseWithCommandLine(ZenExe.FullName), ExportSingleSourceCommandline.ToString());
CommandUtils.RunAndLog(CommandUtils.CmdEnv, ZenExe.FullName, ExportSingleSourceCommandline.ToString(), MaxSuccessCode: 1, Options: CommandUtils.ERunOptions.Default);
ExportIndex = ExportIndex + 1;
}
if ((Parameters.SnapshotDescriptorFile != null) && ExportSources.Any())
{
DirectoryReference.CreateDirectory(Parameters.SnapshotDescriptorFile.Directory);
// Write out a snapshot descriptor
using (JsonWriter Writer = new JsonWriter(Parameters.SnapshotDescriptorFile))
{
Writer.WriteObjectStart();
Writer.WriteArrayStart("snapshots");
ExportIndex = 0;
foreach (ExportSourceData ExportSource in ExportSources)
{
Writer.WriteObjectStart();
IoHash DestinationKeyHash = IoHash.Compute(Encoding.UTF8.GetBytes(ExportKeyIds[ExportIndex]));
Writer.WriteValue("name", ExportKeyIds[ExportIndex]);
Writer.WriteValue("type", "cloud");
Writer.WriteValue("targetplatform", ExportSource.TargetPlatform);
Writer.WriteValue("host", Parameters.DestinationCloudHost);
Writer.WriteValue("namespace", Parameters.DestinationCloudNamespace);
Writer.WriteValue("bucket", BucketName);
Writer.WriteValue("key", DestinationKeyHash.ToString().ToLowerInvariant());
Writer.WriteObjectEnd();
ExportIndex = ExportIndex + 1;
}
Writer.WriteArrayEnd();
Writer.WriteObjectEnd();
}
}
break;
default:
throw new AutomationException("Unknown/invalid/unimplemented destination storage type - {0}", Parameters.DestinationStorageType);
}
return Task.CompletedTask;
}
///
/// Output this task out to an XML writer.
///
public override void Write(XmlWriter Writer)
{
Write(Writer, Parameters);
}
///
/// Find all the tags which are used as inputs to this task
///
/// The tag names which are read by this task
public override IEnumerable FindConsumedTagNames()
{
yield break;
}
///
/// Find all the tags which are modified by this task
///
/// The tag names which are modified by this task
public override IEnumerable FindProducedTagNames()
{
yield break;
}
}
}