Files
UnrealEngineUWP/Engine/Source/Programs/AutomationTool/Scripts/Localisation.Automation.cs
Matthew Griffin bb70b349ce Merging CL 2804086 from //UE4/Release-4.11 to Dev-Main (//UE4/Dev-Main) to isolate copyright update
#lockdown Nick.Penwarden

[CL 2819020 by Matthew Griffin in Main branch]
2016-01-07 08:17:16 -05:00

496 lines
19 KiB
C#

// Copyright 1998-2016 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.IO;
using AutomationTool;
using UnrealBuildTool;
using OneSky;
using EpicGames.OneSkyLocalization.Config;
[Help("Updates the OneSky localization data using the arguments provided.")]
[Help("UEProjectDirectory", "Sub-path to the project we're gathering for (relative to CmdEnv.LocalRoot).")]
[Help("UEProjectName", "Optional name of the project we're gathering for (should match its .uproject file, eg QAGame).")]
[Help("OneSkyConfigName", "Name of the config data to use (see OneSkyConfigHelper).")]
[Help("OneSkyProjectGroupName", "Name of the project group in OneSky.")]
[Help("OneSkyProjectNames", "Comma separated list of the projects in OneSky.")]
[Help("OneSkyBranchSuffix", "Optional suffix to use when uploading the new data to OneSky.")]
class Localise : BuildCommand
{
struct ProjectImportExportInfo
{
/** The destination path for the files containing the gathered data for this project (extracted from its config file - relative to the root working directory for the commandlet) */
public string DestinationPath;
/** The name to use for this projects manifest file (extracted from its config file) */
public string ManifestName;
/** The name to use for this projects archive file (extracted from its config file) */
public string ArchiveName;
/** The name to use for this projects portable object file (extracted from its config file) */
public string PortableObjectName;
/** The native culture for this project (extracted from its config file) */
public string NativeCulture;
/** The cultures to generate for this project (extracted from its config file) */
public List<string> CulturesToGenerate;
/** True if we should use a per-culture directly when importing/exporting */
public bool bUseCultureDirectory;
};
struct ProjectInfo
{
/** The name of this project */
public string ProjectName;
/** Path to this projects localization config files (relative to the root working directory for the commandlet) - ordered so that iterating them runs in the correct order */
public List<string> LocalizationConfigFiles;
/** Config data used by the PO file import process */
public ProjectImportExportInfo ImportInfo;
/** Config data used by the PO file export process */
public ProjectImportExportInfo ExportInfo;
};
private int OneSkyDownloadedPOChangeList = 0;
public override void ExecuteBuild()
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
// Parse out the required command line arguments
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var OneSkyConfigName = ParseParamValue("OneSkyConfigName");
if (OneSkyConfigName == null)
{
throw new AutomationException("Missing required command line argument: 'OneSkyConfigName'");
}
var OneSkyProjectGroupName = ParseParamValue("OneSkyProjectGroupName");
if (OneSkyProjectGroupName == null)
{
throw new AutomationException("Missing required command line argument: 'OneSkyProjectGroupName'");
}
var OneSkyProjectNames = new List<string>();
{
var OneSkyProjectNamesStr = ParseParamValue("OneSkyProjectNames");
if (OneSkyProjectNamesStr == null)
{
throw new AutomationException("Missing required command line argument: 'OneSkyProjectNames'");
}
foreach (var ProjectName in OneSkyProjectNamesStr.Split(','))
{
OneSkyProjectNames.Add(ProjectName.Trim());
}
}
var OneSkyBranchSuffix = ParseParamValue("OneSkyBranchSuffix");
var RootWorkingDirectory = CombinePaths(CmdEnv.LocalRoot, UEProjectDirectory);
// Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on
if (P4Enabled)
{
Log("Sync necessary content to head revision");
P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Config/Localization/...");
P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Content/Localization/...");
}
// Generate the info we need to gather for each project
var ProjectInfos = new List<ProjectInfo>();
foreach (var ProjectName in OneSkyProjectNames)
{
ProjectInfos.Add(GenerateProjectInfo(RootWorkingDirectory, ProjectName));
}
OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find(OneSkyConfigName);
var OneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret);
var OneSkyProjectGroup = GetOneSkyProjectGroup(OneSkyService, OneSkyProjectGroupName);
// Create changelist for backed up POs from OneSky.
if (P4Enabled)
{
OneSkyDownloadedPOChangeList = P4.CreateChange(P4Env.Client, "OneSky downloaded PO backup.");
}
// Export all text from OneSky
foreach (var ProjectInfo in ProjectInfos)
{
ExportOneSkyProjectToDirectory(RootWorkingDirectory, OneSkyService, OneSkyProjectGroup, OneSkyBranchSuffix, ProjectInfo);
}
// Submit changelist for backed up POs from OneSky.
if (P4Enabled)
{
int SubmittedChangeList;
P4.Submit(OneSkyDownloadedPOChangeList, out SubmittedChangeList);
}
// Setup editor arguments for SCC.
string EditorArguments = String.Empty;
if (P4Enabled)
{
EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken());
}
else
{
EditorArguments = String.Format("-SCCProvider={0}", "None");
}
// Setup commandlet arguments for SCC.
string CommandletSCCArguments = String.Empty;
if (P4Enabled) { CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC"; }
if (!AllowSubmit) { CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit"; }
// Execute commandlet for each config in each project.
foreach (var ProjectInfo in ProjectInfos)
{
foreach (var LocalizationConfigFile in ProjectInfo.LocalizationConfigFiles)
{
var CommandletArguments = String.Format("-config={0}", LocalizationConfigFile) + (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments);
Log("Localization for {0} {1}", EditorArguments, CommandletArguments);
Log("Running UE4Editor to generate localization data");
string Arguments = String.Format("{0} -run=GatherText {1} {2}", UEProjectName, EditorArguments, CommandletArguments);
var RunResult = Run(EditorExe, Arguments);
if (RunResult.ExitCode != 0)
{
Console.WriteLine("[ERROR] Error while executing localization commandlet '{0}'", Arguments);
}
}
}
// Upload all text to OneSky
foreach (var ProjectInfo in ProjectInfos)
{
UploadProjectToOneSky(RootWorkingDirectory, OneSkyService, OneSkyProjectGroup, OneSkyBranchSuffix, ProjectInfo);
}
}
private ProjectInfo GenerateProjectInfo(string RootWorkingDirectory, string ProjectName)
{
var ProjectInfo = new ProjectInfo();
ProjectInfo.ProjectName = ProjectName;
ProjectInfo.LocalizationConfigFiles = new List<string>();
// Projects generated by the localization dashboard will use multiple config files that must be run in a specific order
// Older projects (such as the Engine) would use a single config file containing all the steps
// Work out which kind of project we're dealing with...
var MonolithicConfigFile = String.Format(@"Config/Localization/{0}.ini", ProjectName);
var IsMonolithicConfig = File.Exists(CombinePaths(RootWorkingDirectory, MonolithicConfigFile));
if (IsMonolithicConfig)
{
ProjectInfo.LocalizationConfigFiles.Add(MonolithicConfigFile);
ProjectInfo.ImportInfo = GenerateProjectImportExportInfo(RootWorkingDirectory, MonolithicConfigFile);
ProjectInfo.ExportInfo = ProjectInfo.ImportInfo;
}
else
{
var FileSuffixes = new[] {
new { Suffix = "Gather", Required = true },
new { Suffix = "Import", Required = true },
new { Suffix = "Export", Required = true },
new { Suffix = "Compile", Required = true },
new { Suffix = "GenerateReports", Required = false }
};
foreach (var FileSuffix in FileSuffixes)
{
var ModularConfigFile = String.Format(@"Config/Localization/{0}_{1}.ini", ProjectName, FileSuffix.Suffix);
if (File.Exists(CombinePaths(RootWorkingDirectory, ModularConfigFile)))
{
ProjectInfo.LocalizationConfigFiles.Add(ModularConfigFile);
if (FileSuffix.Suffix == "Import")
{
ProjectInfo.ImportInfo = GenerateProjectImportExportInfo(RootWorkingDirectory, ModularConfigFile);
}
else if (FileSuffix.Suffix == "Export")
{
ProjectInfo.ExportInfo = GenerateProjectImportExportInfo(RootWorkingDirectory, ModularConfigFile);
}
}
else if (FileSuffix.Required)
{
throw new AutomationException("Failed to find a required config file! '{0}'", ModularConfigFile);
}
}
}
return ProjectInfo;
}
private ProjectImportExportInfo GenerateProjectImportExportInfo(string RootWorkingDirectory, string LocalizationConfigFile)
{
var ProjectImportExportInfo = new ProjectImportExportInfo();
var LocalizationConfig = new ConfigCacheIni(new FileReference(CombinePaths(RootWorkingDirectory, LocalizationConfigFile)));
if (!LocalizationConfig.GetString("CommonSettings", "DestinationPath", out ProjectImportExportInfo.DestinationPath))
{
throw new AutomationException("Failed to find a required config key! Section: 'CommonSettings', Key: 'DestinationPath', File: '{0}'", LocalizationConfigFile);
}
if (!LocalizationConfig.GetString("CommonSettings", "ManifestName", out ProjectImportExportInfo.ManifestName))
{
throw new AutomationException("Failed to find a required config key! Section: 'CommonSettings', Key: 'ManifestName', File: '{0}'", LocalizationConfigFile);
}
if (!LocalizationConfig.GetString("CommonSettings", "ArchiveName", out ProjectImportExportInfo.ArchiveName))
{
throw new AutomationException("Failed to find a required config key! Section: 'CommonSettings', Key: 'ArchiveName', File: '{0}'", LocalizationConfigFile);
}
if (!LocalizationConfig.GetString("CommonSettings", "PortableObjectName", out ProjectImportExportInfo.PortableObjectName))
{
throw new AutomationException("Failed to find a required config key! Section: 'CommonSettings', Key: 'PortableObjectName', File: '{0}'", LocalizationConfigFile);
}
if (!LocalizationConfig.GetString("CommonSettings", "NativeCulture", out ProjectImportExportInfo.NativeCulture))
{
throw new AutomationException("Failed to find a required config key! Section: 'CommonSettings', Key: 'NativeCulture', File: '{0}'", LocalizationConfigFile);
}
if (!LocalizationConfig.GetArray("CommonSettings", "CulturesToGenerate", out ProjectImportExportInfo.CulturesToGenerate))
{
throw new AutomationException("Failed to find a required config key! Section: 'CommonSettings', Key: 'CulturesToGenerate', File: '{0}'", LocalizationConfigFile);
}
if (!LocalizationConfig.GetBool("CommonSettings", "bUseCultureDirectory", out ProjectImportExportInfo.bUseCultureDirectory))
{
// bUseCultureDirectory is optional, default is true
ProjectImportExportInfo.bUseCultureDirectory = true;
}
return ProjectImportExportInfo;
}
private static ProjectGroup GetOneSkyProjectGroup(OneSkyService OneSkyService, string ProjectGroupName)
{
var OneSkyProjectGroup = OneSkyService.ProjectGroups.FirstOrDefault(g => g.Name == ProjectGroupName);
if (OneSkyProjectGroup == null)
{
OneSkyProjectGroup = new ProjectGroup(ProjectGroupName, "en");
OneSkyService.ProjectGroups.Add(OneSkyProjectGroup);
}
return OneSkyProjectGroup;
}
private static OneSky.Project GetOneSkyProject(OneSkyService OneSkyService, string GroupName, string ProjectName, string ProjectDescription = "")
{
var OneSkyProjectGroup = GetOneSkyProjectGroup(OneSkyService, GroupName);
OneSky.Project OneSkyProject = OneSkyProjectGroup.Projects.FirstOrDefault(p => p.Name == ProjectName);
if (OneSkyProject == null)
{
ProjectType projectType = OneSkyService.ProjectTypes.First(pt => pt.Code == "website");
OneSkyProject = new OneSky.Project(ProjectName, ProjectDescription, projectType);
OneSkyProjectGroup.Projects.Add(OneSkyProject);
}
return OneSkyProject;
}
private void ExportOneSkyProjectToDirectory(string RootWorkingDirectory, OneSkyService OneSkyService, ProjectGroup OneSkyProjectGroup, string OneSkyBranchSuffix, ProjectInfo ProjectInfo)
{
var OneSkyFileName = GetOneSkyFilename(ProjectInfo.ImportInfo.PortableObjectName, OneSkyBranchSuffix);
var OneSkyProject = GetOneSkyProject(OneSkyService, OneSkyProjectGroup.Name, ProjectInfo.ProjectName);
var OneSkyFile = OneSkyProject.UploadedFiles.FirstOrDefault(f => f.Filename == OneSkyFileName);
//Export
if (OneSkyFile != null)
{
var CulturesToExport = new List<string>();
foreach (var OneSkyCulture in OneSkyProject.EnabledCultures)
{
// Only export the OneSky cultures that we care about for this project
if (ProjectInfo.ImportInfo.CulturesToGenerate.Contains(OneSkyCulture))
{
CulturesToExport.Add(OneSkyCulture);
}
}
ExportOneSkyFileToDirectory(OneSkyFile, new DirectoryInfo(CombinePaths(RootWorkingDirectory, ProjectInfo.ImportInfo.DestinationPath)), ProjectInfo.ImportInfo.PortableObjectName, CulturesToExport, ProjectInfo.ImportInfo.bUseCultureDirectory);
}
}
private void ExportOneSkyFileToDirectory(UploadedFile OneSkyFile, DirectoryInfo DestinationDirectory, string DestinationFilename, IEnumerable<string> Cultures, bool bUseCultureDirectory)
{
foreach (var Culture in Cultures)
{
var CultureDirectory = (bUseCultureDirectory) ? new DirectoryInfo(Path.Combine(DestinationDirectory.FullName, Culture)) : DestinationDirectory;
if (!CultureDirectory.Exists)
{
CultureDirectory.Create();
}
using (var MemoryStream = new MemoryStream())
{
var ExportTranslationState = OneSkyFile.ExportTranslation(Culture, MemoryStream).Result;
if (ExportTranslationState == UploadedFile.ExportTranslationState.Success)
{
var ExportFile = new FileInfo(Path.Combine(CultureDirectory.FullName, DestinationFilename));
// Write out the updated PO file so that the gather commandlet will import the new data from it
{
var ExportFileWasReadOnly = false;
if (ExportFile.Exists)
{
// We're going to clobber the existing PO file, so make sure it's writable (it may be read-only if in Perforce)
ExportFileWasReadOnly = ExportFile.IsReadOnly;
ExportFile.IsReadOnly = false;
}
MemoryStream.Position = 0;
using (Stream FileStream = ExportFile.OpenWrite())
{
MemoryStream.CopyTo(FileStream);
Console.WriteLine("[SUCCESS] Exporting: '{0}' as '{1}' ({2})", OneSkyFile.Filename, ExportFile.FullName, Culture);
}
if (ExportFileWasReadOnly)
{
ExportFile.IsReadOnly = true;
}
}
// Also update the back-up copy so we can diff against what we got from OneSky, and what the gather commandlet produced
{
var ExportFileCopy = new FileInfo(Path.Combine(ExportFile.DirectoryName, String.Format("{0}_FromOneSky{1}", Path.GetFileNameWithoutExtension(ExportFile.Name), ExportFile.Extension)));
var ExportFileCopyWasReadOnly = false;
if (ExportFileCopy.Exists)
{
// We're going to clobber the existing PO file, so make sure it's writable (it may be read-only if in Perforce)
ExportFileCopyWasReadOnly = ExportFileCopy.IsReadOnly;
ExportFileCopy.IsReadOnly = false;
}
ExportFile.CopyTo(ExportFileCopy.FullName, true);
if (ExportFileCopyWasReadOnly)
{
ExportFileCopy.IsReadOnly = true;
}
// Add/check out backed up POs from OneSky.
if (P4Enabled)
{
UE4Build.AddBuildProductsToChangelist(OneSkyDownloadedPOChangeList, new List<string>() { ExportFileCopy.FullName });
}
}
}
else if (ExportTranslationState == UploadedFile.ExportTranslationState.NoContent)
{
Console.WriteLine("[WARNING] Exporting: '{0}' ({1}) has no translations!", OneSkyFile.Filename, Culture);
}
else
{
Console.WriteLine("[FAILED] Exporting: '{0}' ({1})", OneSkyFile.Filename, Culture);
}
}
}
}
private void UploadProjectToOneSky(string RootWorkingDirectory, OneSkyService OneSkyService, ProjectGroup OneSkyProjectGroup, string OneSkyBranchSuffix, ProjectInfo ProjectInfo)
{
var OneSkyProject = GetOneSkyProject(OneSkyService, OneSkyProjectGroup.Name, ProjectInfo.ProjectName);
Func<string, FileInfo> GetPathForCulture = (string Culture) =>
{
if (ProjectInfo.ExportInfo.bUseCultureDirectory)
{
return new FileInfo(Path.Combine(RootWorkingDirectory, ProjectInfo.ExportInfo.DestinationPath, Culture, ProjectInfo.ExportInfo.PortableObjectName));
}
else
{
return new FileInfo(Path.Combine(RootWorkingDirectory, ProjectInfo.ExportInfo.DestinationPath, ProjectInfo.ExportInfo.PortableObjectName));
}
};
// Upload the .po file for the native culture first
UploadFileToOneSky(OneSkyProject, OneSkyBranchSuffix, GetPathForCulture(ProjectInfo.ExportInfo.NativeCulture), ProjectInfo.ExportInfo.NativeCulture);
// Upload the remaining .po files for the other cultures
foreach (var Culture in ProjectInfo.ExportInfo.CulturesToGenerate)
{
// Skip native culture as we uploaded it above
if (Culture != ProjectInfo.ExportInfo.NativeCulture)
{
UploadFileToOneSky(OneSkyProject, OneSkyBranchSuffix, GetPathForCulture(Culture), Culture);
}
}
}
private void UploadFileToOneSky(OneSky.Project OneSkyProject, string OneSkyBranchSuffix, FileInfo FileToUpload, string Culture)
{
using (var FileStream = FileToUpload.OpenRead())
{
// Read the BOM
var UTF8BOM = new byte[3];
FileStream.Read(UTF8BOM, 0, 3);
// We want to ignore the utf8 BOM
if (UTF8BOM[0] != 0xef || UTF8BOM[1] != 0xbb || UTF8BOM[2] != 0xbf)
{
FileStream.Position = 0;
}
var OneSkyFileName = GetOneSkyFilename(Path.GetFileName(FileToUpload.FullName), OneSkyBranchSuffix);
Console.WriteLine("Uploading: '{0}' as '{1}' ({2})", FileToUpload.FullName, OneSkyFileName, Culture);
var UploadedFile = OneSkyProject.Upload(OneSkyFileName, FileStream, Culture).Result;
if (UploadedFile == null)
{
Console.WriteLine("[FAILED] Uploading: '{0}' ({1})", FileToUpload.FullName, Culture);
}
else
{
Console.WriteLine("[SUCCESS] Uploading: '{0}' ({1})", FileToUpload.FullName, Culture);
}
}
}
private static string GetOneSkyFilename(string BaseFilename, string OneSkyBranchSuffix)
{
var OneSkyFileName = BaseFilename;
if (!String.IsNullOrEmpty(OneSkyBranchSuffix))
{
// Apply the branch suffix. OneSky will take care of merging the files from different branches together.
var OneSkyFileNameWithSuffix = Path.GetFileNameWithoutExtension(OneSkyFileName) + "_" + OneSkyBranchSuffix + Path.GetExtension(OneSkyFileName);
OneSkyFileName = OneSkyFileNameWithSuffix;
}
return OneSkyFileName;
}
}