izzy/UnrealEngineUWP
0
0
Fork 0
mirror of https://github.com/izzy2lost/UnrealEngineUWP.git synced 2026-03-26 18:15:20 -07:00
Code Issues Packages Projects Releases Wiki Activity

Remove the "Distill" program, which isn't being used any more.

Browse Source 
[CL 2498529 by Ben Marsh in Main branch]
This commit is contained in:
Ben Marsh
2015-04-01 09:13:16 -04:00
committed by Ben.Marsh@epicgames.com
parent 41a8bd5eee
commit ae84053fc9
12 changed files with 0 additions and 1724 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Engine/Source/Programs/AutomationTool/Scripts/GUBP.Automation.cs
View File

@@ -1150,7 +1150,6 @@ public class GUBP : BuildCommand
Agenda.DotNetProjects.AddRange(
new string[]
{
CombinePaths(@"Engine\Source\Programs\Distill\Distill.csproj"),
CombinePaths(@"Engine\Source\Programs\NotForLicensees\CrashReportServer\CrashReportCommon\CrashReportCommon.csproj"),
CombinePaths(@"Engine\Source\Programs\NotForLicensees\CrashReportServer\CrashReportReceiver\CrashReportReceiver.csproj"),
CombinePaths(@"Engine\Source\Programs\NotForLicensees\CrashReportServer\CrashReportProcess\CrashReportProcess.csproj"),

116
Engine/Source/Programs/Distill/Distill.csproj
View File

@@ -1,116 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
<ProductVersion>8.0.30703</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{586471A8-BA8A-4101-B160-EEF0E5A8144D}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Tools.Distill</RootNamespace>
<AssemblyName>Distill</AssemblyName>
<TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
<TargetFrameworkProfile>Client</TargetFrameworkProfile>
<FileAlignment>512</FileAlignment>
<IsWebBootstrapper>false</IsWebBootstrapper>
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|AnyCPU'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>..\..\..\Binaries\DotNET\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>AnyCPU</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisIgnoreBuiltInRuleSets>true</CodeAnalysisIgnoreBuiltInRuleSets>
<CodeAnalysisIgnoreBuiltInRules>true</CodeAnalysisIgnoreBuiltInRules>
<DocumentationFile>..\..\..\Binaries\DotNET\Distill.xml</DocumentationFile>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Development|AnyCPU'">
<OutputPath>..\..\..\Binaries\DotNET\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<DocumentationFile>..\..\..\Binaries\DotNET\Distill.xml</DocumentationFile>
<Optimize>true</Optimize>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<DebugType>pdbonly</DebugType>
<PlatformTarget>AnyCPU</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<ItemGroup>
<Reference Include="Ionic.Zip.Reduced">
<HintPath>..\..\..\Binaries\DotNET\Ionic.Zip.Reduced.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\DotNETCommon\MetaData.cs">
<Link>Properties\MetaData.cs</Link>
</Compile>
<Compile Include="Private\Crypto.cs" />
<Compile Include="Private\Distill.cs" />
<Compile Include="Private\FileUtils.cs" />
<Compile Include="Private\FTP.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Private\Selection.cs" />
<Compile Include="Private\Settings.cs" />
<Compile Include="Private\TOC.cs" />
<Compile Include="Private\Zip.cs" />
</ItemGroup>
<ItemGroup>
<BootstrapperPackage Include=".NETFramework,Version=v4.0,Profile=Client">
<Visible>False</Visible>
<ProductName>Microsoft .NET Framework 4 Client Profile %28x86 and x64%29</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DotNETCommon\DotNETUtilities\DotNETUtilities.csproj">
<Project>{5d7d66e8-8c76-4af9-b3ec-2ef03421d730}</Project>
<Name>DotNETUtilities</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

73
Engine/Source/Programs/Distill/Private/Crypto.cs
View File

@@ -1,73 +0,0 @@
// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Text;
namespace Tools.Distill
{
/// <summary>A helper class to handle SHA256 creation and x509 signature handling</summary>
public class Crypto
{
private static List<string> ValidSerialNumbers = new List<string>()
{
// Serial numbers of X509 digital signatures that we allow to be packaged
"3AF820A6907699580410055228ECADDF", // Nvidia1
"534ABED0BE56D9840DD12DDB84F8B031", // Nvidia2
"610F784D000000000003", // Microsoft1
"61062781000000000008", // Microsoft2
"6101CF3E00000000000F", // Microsoft3
"6101B29B000000000015", // Microsoft4
"68697535E381FE1F91C1153528146A27", // Valve
"7BF6326F70CBEC340BF2D1868FE65B1E", // Valve
"19599BE196632020F023C96F2F3904D6", // AMD
"479525AB0000000001C8", // Intel
"64C4C6DAAEE1C43378152EFEC163241C", // Imagination Technologies
"00B743783950C32FFA9842A5DC404294", // Perforce
"1C16CDDAD8961907E5704B82D9A15842", // Epic Games Inc. (old)
"3E1338505D2C3D34E2DF71AE64C5C24F", // Epic Games Inc. (new)
};
/// <summary>Validate the digital signature of the file matches one of the whitelisted serial numbers</summary>
/// <param name="Info">FileInfo of the file to verify</param>
/// <returns>True if the digital signature has a whitelisted serial number</returns>
public static bool ValidateDigitalSignature( FileInfo Info )
{
try
{
X509Certificate Certificate = X509Certificate.CreateFromSignedFile( Info.FullName );
if( Certificate != null )
{
if( ValidSerialNumbers.Contains( Certificate.GetSerialNumberString() ) )
{
return true;
}
}
}
catch
{
// Any exception means that either the file isn't signed or has an invalid certificate
}
return false;
}
/// <summary>Create a SHA256 checksum of the file contents</summary>
/// <param name="Info">FileInfo of the file to checksum</param>
/// <returns>A 32 byte array of the SHA256 checksum</returns>
/// <remarks>Microsoft no longer recommend using SHA1 as it is insecure</remarks>
public static byte[] CreateChecksum( FileInfo Info )
{
SHA256CryptoServiceProvider Hasher = new SHA256CryptoServiceProvider();
FileStream Stream = Info.OpenRead();
byte[] HashData = Hasher.ComputeHash( Stream );
Stream.Close();
return HashData;
}
}
}

433
Engine/Source/Programs/Distill/Private/Distill.cs
View File

@@ -1,433 +0,0 @@
// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using Ionic.Zip;
using Tools.DotNETCommon.XmlHandler;
namespace Tools.Distill
{
/// <summary>
/// Distill - a build copying utility
/// </summary>
class NamespaceDoc
{
}
/// <summary>A container for all the available options</summary>
public class DistillOptions
{
/// <summary>The name of the game to use (including the game extension)</summary>
public string Game = "Engine";
/// <summary>A list of all games, excluding the selected game. Games are automatically discovered.</summary>
public List<string> NotGames = null;
/// <summary>The source of the file soup</summary>
public string Source = "";
/// <summary>The destination of the files</summary>
public string Destination = "";
/// <summary>The destination zip file</summary>
public string ZipName = "";
/// <summary>The FTP site to upload to</summary>
public string FTPSite = "";
/// <summary>The folder on the FTP site to upload to</summary>
public string FTPFolder = "";
/// <summary>The FTP user to upload to the FTP site</summary>
public string FTPUser = "";
/// <summary>The FTP password for the FTP user</summary>
public string FTPPass = "";
/// <summary>The platform to distill files for</summary>
public string Platform = "Windows";
/// <summary>Additional platforms which should be included</summary>
public List<string> IncPlatforms = new List<string>();
/// <summary>A list of all the platforms, excluding the selected platform. Defined in the KnownPlatforms section of the settings xml</summary>
public List<string> NotPlatforms = null;
/// <summary>The language to distill files for</summary>
public string Region = "INT";
/// <summary>A list of all languages, excluding the selected language. Languages are automatically discovered.</summary>
public List<string> NotLanguages = null;
/// <summary>The name of the set of tags to manipulate files</summary>
public string TagSet = "Default";
/// <summary>The xml file to load</summary>
public string DistillSettings;
/// <summary></summary>
public bool bAuthenticate = false;
/// <summary></summary>
public bool bChecksum = false;
/// <summary></summary>
public bool bForce = false;
/// <summary></summary>
public bool bLog = false;
/// <summary></summary>
public bool bNoCopy = false;
/// <summary></summary>
public bool bTOC = false;
/// <summary></summary>
public bool bVerify = false;
/// <summary></summary>
public bool bWhitelistSignatures = false;
}
/// <summary>A utility assembly to handle copying, zipping, authenticating, and FTPing of builds</summary>
public partial class Distill
{
/// <summary>A parsed version of the commandline options</summary>
public static DistillOptions Options = new DistillOptions();
private static void ShowUsage()
{
Log( "Distills a build out of the file soup to and/or from one or more locations.", ConsoleColor.Gray );
Log( "", ConsoleColor.Gray );
Log( "Distill.exe [-Options] [-Game=<Name>] [-Source=<Path>] [-Destination=<Path>]", ConsoleColor.Gray );
Log( "", ConsoleColor.Gray );
Log( "Options:", ConsoleColor.Gray );
Log( " -a (-Authenticate) : Verify the local files have the same checksums as in the TOC", ConsoleColor.Gray );
Log( " -c (-Checksum) : Generate SHA256 checksums when creating the TOC", ConsoleColor.Gray );
Log( " -Destination=<Name> : Destination UNC path", ConsoleColor.Gray );
Log( " -f (-Force) : Force copying of all files regardless of time stamp", ConsoleColor.Gray );
Log( " -FTPSite=<Name> : FTP site to send to", ConsoleColor.Gray );
Log( " -FTPFolder=<Name> : FTP root folder", ConsoleColor.Gray );
Log( " -FTPUser=<Name> : FTP user for the FTP site", ConsoleColor.Gray );
Log( " -FTPPass=<Name> : FTP password for the user", ConsoleColor.Gray );
Log( " -Game=<Name> : Name of the game to be copied (typically ending in 'Game')", ConsoleColor.Gray );
Log( " -l (-Log) : More verbose logging output", ConsoleColor.Gray );
Log( " -n (-NoCopy) : Do not sync files, preview only", ConsoleColor.Gray );
Log( " -TOC : Save a Table of Contents (TOC) to disk.", ConsoleColor.Gray );
Log( " -Platform=<Name> : Specify platform <Platform> to be used", ConsoleColor.Gray);
Log( " Can be one of: Windows, IPhone, Android etc. Default: Windows.", ConsoleColor.Gray);
Log( " -IncPlatform=<Name> : Specify platform which should not be excluded through %NOTPLATFORM% macros", ConsoleColor.Gray);
Log( " -Region=<Name> : Three letter code determining packages to copy [Default is INT]", ConsoleColor.Gray);
Log( " : Used for copying loc packages, all packages of the type _LOC.upk", ConsoleColor.Gray );
Log( " -Source=<Name> : Specifies an alternate source for the data.", ConsoleColor.Gray );
Log( " -TagSet=<Name> : Only sync files in tag set <TagSet>. See Distill.xml for the tag sets.", ConsoleColor.Gray );
Log( " -v (-Verify) : Verify that the SHAs match between source and destination", ConsoleColor.Gray );
Log( " Use in combination with -a", ConsoleColor.Gray );
Log( " -w (-Whitelist) : Only include exes and dlls with a whitelisted digital signature", ConsoleColor.Gray );
Log( " -ZipName=<Name> : Destination zip file", ConsoleColor.Gray );
Log( "", ConsoleColor.Gray );
}
private static void ParseArguments( string[] Arguments )
{
for( int ArgumentIndex = 0; ArgumentIndex < Arguments.Length; ArgumentIndex++ )
{
if( Arguments[ArgumentIndex].Contains( "=" ) )
{
string[] Pair = Arguments[ArgumentIndex].Split( "=".ToCharArray() );
if( Pair.Length == 2 )
{
switch( Pair[0].ToLower() )
{
case "-distillsettings":
Options.DistillSettings = Pair[1];
break;
case "-destination":
Options.Destination = Pair[1];
break;
case "-ftpsite":
Options.FTPSite = Pair[1];
break;
case "-ftpfolder":
Options.FTPFolder = Pair[1];
break;
case "-game":
Options.Game = Pair[1];
break;
case "-ftpuser":
Options.FTPUser = Pair[1];
break;
case "-ftppass":
Options.FTPPass = Pair[1];
break;
case "-platform":
Options.Platform = Pair[1];
break;
case "-incplatform":
Options.IncPlatforms.Add(Pair[1]);
break;
case "-region":
Options.Region = Pair[1];
break;
case "-source":
Options.Source = Pair[1];
break;
case "-tagset":
Options.TagSet = Pair[1];
break;
case "-zipname":
Options.ZipName = Pair[1];
break;
default:
Warning( "Unknown option: " + Pair[0] );
break;
}
}
}
else
{
switch( Arguments[ ArgumentIndex ].ToLower() )
{
case "-a":
Options.bAuthenticate = true;
break;
case "-c":
Options.bChecksum = true;
break;
case "-f":
Options.bForce = true;
break;
case "-l":
Options.bLog = true;
break;
case "-n":
Options.bNoCopy = true;
break;
case "-toc":
Options.bTOC = true;
break;
case "-v":
Options.bVerify = true;
break;
case "-w":
Options.bWhitelistSignatures = true;
break;
default:
Warning( "Unknown option: " + Arguments[ ArgumentIndex ] );
break;
}
}
}
}
private static bool ValidateOptions( DistillSettings Settings )
{
if( Settings.FileGroups == null || Settings.KnownPlatforms == null || Settings.TagSets == null )
{
Error( "Failed to load configuration file" );
return false;
}
// Fix the case of the platform
foreach( PlatformInfo Platform in Settings.KnownPlatforms )
{
if( Platform.Name.ToUpper() == Options.Platform.ToUpper() )
{
Options.Platform = Platform.Name;
break;
}
}
if( ( Options.bAuthenticate || Options.bVerify ) && !Options.bChecksum )
{
Warning( " ... enabling checksums (-c) to allow verification and authentication" );
Options.bChecksum = true;
}
return true;
}
/// <summary>Write a line of important text to the console</summary>
/// <param name="Line">The line of text to write to the console</param>
/// <param name="Color">The colour to use when writing the text</param>
public static void Log( string Line, ConsoleColor Color )
{
Console.ForegroundColor = Color;
Console.WriteLine( Line );
}
/// <summary>Write a line of important text to the console, but suppress the line feed</summary>
/// <param name="Line">The line of text to write to the console</param>
/// <param name="Color">The colour to use when writing the text</param>
public static void ProgressLog( string Line, ConsoleColor Color )
{
Console.ForegroundColor = Color;
Console.Write( "\r" + Line );
}
/// <summary>Write a line of text to the console if verbose logging is enabled</summary>
/// <param name="Line">The line of text to write to the console</param>
/// <param name="Color">The colour to use when writing the text</param>
public static void DetailLog( string Line, ConsoleColor Color )
{
if( Options.bLog )
{
Console.ForegroundColor = Color;
Console.WriteLine( Line );
}
}
/// <summary>Write a warning line to the console</summary>
/// <param name="Line">The warning to report</param>
public static void Warning( string Line )
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine( "WARNING: " + Line );
}
/// <summary>Write an error line o the console</summary>
/// <param name="Line">The error to report</param>
public static void Error( string Line )
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine( "ERROR: " + Line );
ErrorCode = 1;
}
/// <summary>Authenticate the local files against the checksums in the TOC</summary>
/// <param name="TableOfContents">Freshly created TOC from the local file system</param>
public static void Authenticate( TOC TableOfContents )
{
string TOCFileName = Path.Combine( Options.Source, Options.Game, "TOC.xml" );
TOC OriginalTableOfContents = XmlHandler.ReadXml<TOC>( TOCFileName );
if( OriginalTableOfContents.Entries != null )
{
OriginalTableOfContents.Compare( TableOfContents );
}
else
{
Error( "Failed to load TOC: " + TOCFileName );
}
}
static int ErrorCode = 0;
/// <summary>The main entry point of the program</summary>
/// <param name="Arguments">The arguments passed on on the commandline</param>
/// <returns>Zero on success, non zero for failure</returns>
private static int Main( string[] Arguments )
{
if( Arguments.Length == 0 )
{
ShowUsage();
return 1;
}
// Set the current directory to the root of the branch
Environment.CurrentDirectory = Path.Combine( Environment.CurrentDirectory, "..", "..", ".." );
Options.Source = Environment.CurrentDirectory;
Options.Destination = Environment.CurrentDirectory;
// Remember console color for restoring on exit
ConsoleColor OriginalConsoleColor = Console.ForegroundColor;
// Parse the command line for settings
ParseArguments( Arguments );
// Load the Distill.xml file and the game specific version
string SettingsLocation = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), Options.DistillSettings);
if (File.Exists(SettingsLocation) == false)
{
Error("Unable to find distill file " + Options.DistillSettings);
Console.ForegroundColor = OriginalConsoleColor;
return -1;
}
DistillSettings Settings = XmlHandler.ReadXml<DistillSettings>( SettingsLocation );
// Find the known languages based on the folders in Engine Localization
Settings.KnownLanguages = FindKnownLanguages();
// Ensure the settings are valid
if( !ValidateOptions( Settings ) )
{
Console.ForegroundColor = OriginalConsoleColor;
return -1;
}
// Get the set of tags we wish to copy
TagSet Set = GetTagSet( Settings );
if( Set == null )
{
Console.ForegroundColor = OriginalConsoleColor;
return -1;
}
// Get a list of filesets that includes all the unexpanded macros
List<FileSet> DecoratedFileSets = GetFileSets( Settings, Set );
if( DecoratedFileSets.Count == 0 )
{
Error( "No file sets for game, platform, tagset combination!" );
Console.ForegroundColor = OriginalConsoleColor;
return -1;
}
// Expand out all the macros
List<FileSet> FileSets = ExpandMacros( DecoratedFileSets, Settings );
// Get the files referenced by the filesets
List<FileInfo> Files = GetFiles( FileSets );
// Create a TOC
TOC TableOfContents = new TOC( Files );
Files = null;
if( Options.bAuthenticate )
{
// If we're authenticating, compare the newly created TOC from files with the existing one on disk
Authenticate( TableOfContents );
}
else
{
// Only write a TOC if we're copying locally to somewhere else (the most common case)
string TOCFileName = Path.Combine( Options.Source, Options.Game, "TOC.xml" );
if( Options.bTOC && (Options.Source == Environment.CurrentDirectory) )
{
Log( " ... writing TOC: " + TOCFileName, ConsoleColor.Cyan );
XmlHandler.WriteXml<TOC>( TableOfContents, TOCFileName, "" );
}
// Copy the TOC if it exists
if (Options.bTOC)
{
FileInfo TOCFileInfo = new FileInfo(TOCFileName);
if (TOCFileInfo.Exists)
{
TableOfContents.Entries.Add(new TOCEntry(TOCFileInfo));
}
else
{
Error(" ... Expected a TOC but there isn't one to copy: " + TOCFileName);
}
}
// Copy files
ZipFiles( TableOfContents );
CopyFiles( TableOfContents );
FTPFiles( TableOfContents );
Log( "Distill process successful!", ConsoleColor.Green );
}
// Restore console color
Console.ForegroundColor = OriginalConsoleColor;
return ErrorCode;
}
}
}

192
Engine/Source/Programs/Distill/Private/FTP.cs
View File

@@ -1,192 +0,0 @@
// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using Tools.DotNETCommon.ThreadSafeQueue;
namespace Tools.Distill
{
public partial class Distill
{
private class FTPFilePacket
{
public FileInfo SourceFile = null;
public string DestinationFile = null;
public FTPFilePacket( FileInfo InSourceFile, string InDestinationFile )
{
SourceFile = InSourceFile;
DestinationFile = InDestinationFile;
}
}
private static bool bFTPingFiles = true;
private static long TotalBytesUploaded = 0;
private static Hashtable Folders = new Hashtable();
private static ThreadSafeQueue<FTPFilePacket> FTPFilePackets = new ThreadSafeQueue<FTPFilePacket>();
private static void CreateFolder( string DirectoryName )
{
Uri DestURI = new Uri( "ftp://" + Options.FTPSite + "/" + DirectoryName );
FtpWebRequest Request = ( FtpWebRequest )FtpWebRequest.Create( DestURI );
// Set the credentials required to connect to the FTP server
Request.Credentials = new NetworkCredential( Options.FTPUser, Options.FTPPass );
// Automatically close the connection on completion
Request.KeepAlive = true;
// Set to upload a file
Request.Method = WebRequestMethods.Ftp.MakeDirectory;
WebResponse Response = Request.GetResponse();
}
private static void EnsureRemoteFolderExists( string DirectoryName )
{
if( DirectoryName.Contains( "/" ) )
{
EnsureRemoteFolderExists( Path.GetDirectoryName( DirectoryName ).Replace( "\\", "/" ) );
}
if( !Folders.Contains( DirectoryName ) )
{
try
{
Folders[DirectoryName] = true;
DetailLog( "Creating folder: " + DirectoryName, ConsoleColor.DarkMagenta );
CreateFolder( DirectoryName );
}
catch
{
}
}
}
private static Stream GetWebRequestStream( FileInfo SourceFile, string DestinationName )
{
Uri DestURI = new Uri( "ftp://" + Options.FTPSite + "/" + DestinationName );
FtpWebRequest Request = ( FtpWebRequest )FtpWebRequest.Create( DestURI );
// Set the credentials required to connect to the FTP server
Request.Credentials = new NetworkCredential( Options.FTPUser, Options.FTPPass );
// Automatically close the connection on completion
Request.KeepAlive = true;
// Set to upload a file
Request.Method = WebRequestMethods.Ftp.UploadFile;
// Send a binary file
Request.UseBinary = true;
// Amount of data to upload
Request.ContentLength = SourceFile.Length;
return Request.GetRequestStream();
}
private static long UploadFile( FileInfo SourceFile, string DestinationName )
{
FileStream Source = SourceFile.OpenRead();
Stream Destination = GetWebRequestStream( SourceFile, DestinationName );
int MaxBufferLength = 65536;
byte[] Buffer = new byte[MaxBufferLength];
int BufferLength = Source.Read( Buffer, 0, MaxBufferLength );
while( BufferLength > 0 )
{
Destination.Write( Buffer, 0, BufferLength );
TotalBytesUploaded += BufferLength;
BufferLength = Source.Read( Buffer, 0, MaxBufferLength );
}
Destination.Close();
Source.Close();
return SourceFile.Length;
}
private static void FTPFilesThreadProc()
{
while( bFTPingFiles )
{
while( FTPFilePackets.Count > 0 )
{
FTPFilePacket Packet = null;
try
{
Packet = FTPFilePackets.Dequeue();
DetailLog( " ... FTPing: " + Packet.SourceFile.FullName, ConsoleColor.DarkYellow );
EnsureRemoteFolderExists( Path.GetDirectoryName( Packet.DestinationFile ).Replace( "\\", "/" ) );
UploadFile( Packet.SourceFile, Packet.DestinationFile );
}
catch( Exception Ex )
{
Warning( "Failed to FTP: " + Packet.SourceFile.FullName + " (" + Ex.Message + ") -- trying again!" );
Packet.SourceFile.Refresh();
FTPFilePackets.Enqueue( Packet );
}
}
}
}
/// <summary>
///
/// </summary>
/// <param name="TableOfContents"></param>
public static void FTPFiles( TOC TableOfContents )
{
if( Options.FTPSite.Length > 0 && Options.FTPUser.Length > 0 )
{
int TotalFilesUploaded = TableOfContents.Entries.Count;
TotalBytesUploaded = 0;
bCopyingFiles = true;
Thread FTPFilesThread = new Thread( FTPFilesThreadProc );
FTPFilesThread.Start();
foreach( TOCEntry Entry in TableOfContents.Entries )
{
// Handle file copies
FileInfo SourceFile = Entry.Info;
string DestFile = Path.Combine( Options.FTPFolder, Entry.Name ).Replace( "\\", "/" );
// Create copy packet and add to queue
FTPFilePacket Packet = new FTPFilePacket( SourceFile, DestFile );
FTPFilePackets.Enqueue( Packet );
}
while( FTPFilePackets.Count > 0 )
{
if( !Options.bLog )
{
ProgressLog( " ... waiting for " + FTPFilePackets.Count + " FTPs (" + TotalBytesUploaded + " uploaded) ", ConsoleColor.Cyan );
}
Thread.Sleep( 100 );
}
if( !Options.bLog )
{
ProgressLog( " ... waiting for " + FTPFilePackets.Count + " FTPs (" + TotalBytesUploaded + " bytes uploaded) ", ConsoleColor.Cyan );
}
bFTPingFiles = false;
Log( "", ConsoleColor.Green );
Log( "Completed FTPing " + TotalFilesUploaded + " files, totaling " + TotalBytesUploaded + " bytes", ConsoleColor.Green );
}
}
}
}

255
Engine/Source/Programs/Distill/Private/FileUtils.cs
View File

@@ -1,255 +0,0 @@
// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using Tools.DotNETCommon.ThreadSafeQueue;
namespace Tools.Distill
{
public partial class Distill
{
/// <summary>Find all languages based on a file system search</summary>
/// <returns>An array of languages found</returns>
public static string[] FindKnownLanguages()
{
DetailLog( "Finding known languages ...", ConsoleColor.Blue );
List<string> KnownLanguages = new List<string>();
DirectoryInfo DirInfo = new DirectoryInfo( Path.Combine( Options.Source, "Engine/Content/Localization" ) );
if( DirInfo.Exists )
{
foreach( DirectoryInfo SubDirInfo in DirInfo.GetDirectories( "???" ) )
{
if( SubDirInfo.GetFiles( Path.ChangeExtension( "Core", SubDirInfo.Name ) ).Length == 0 )
{
continue;
}
string KnownLanguage = SubDirInfo.Name.ToUpper();
KnownLanguages.Add( KnownLanguage );
DetailLog( " ... adding known language: " + KnownLanguage, ConsoleColor.Blue );
}
}
if( !KnownLanguages.Contains( "INT" ) )
{
KnownLanguages.Add( "INT" );
DetailLog( " ... INT not found; adding INT to known languages ", ConsoleColor.Blue );
}
return KnownLanguages.ToArray();
}
private static bool CopyRequired( FileInfo SourceFile, FileInfo DestFile )
{
if( Options.bNoCopy )
{
return false;
}
if( Options.bForce )
{
return true;
}
if( !DestFile.Exists )
{
return true;
}
if( SourceFile.Length != DestFile.Length )
{
return true;
}
// Allow for the slop in OSX file time stamps
if( SourceFile.LastWriteTimeUtc > DestFile.LastWriteTimeUtc.AddSeconds( 2 ) )
{
return true;
}
return false;
}
private class CopyFilePacket
{
public FileInfo SourceFile = null;
public FileInfo DestinationFile = null;
public string Hash = "";
public CopyFilePacket( FileInfo InSourceFile, FileInfo InDestinationFile, string InHash )
{
SourceFile = InSourceFile;
DestinationFile = InDestinationFile;
Hash = InHash;
}
}
private static bool bCopyingFiles = true;
private static bool bVerifyingFiles = true;
private static ThreadSafeQueue<CopyFilePacket> CopyFilePackets = new ThreadSafeQueue<CopyFilePacket>();
private static ThreadSafeQueue<CopyFilePacket> VerifyFilePackets = new ThreadSafeQueue<CopyFilePacket>();
private static void EnsureFolderExists( DirectoryInfo DirInfo )
{
if( DirInfo.Parent != null )
{
EnsureFolderExists( DirInfo.Parent );
}
if( !DirInfo.Exists )
{
DirInfo.Create();
}
}
private static void CopyFilesThreadProc()
{
while( bCopyingFiles )
{
while( CopyFilePackets.Count > 0 )
{
CopyFilePacket Packet = null;
try
{
Packet = CopyFilePackets.Dequeue();
DetailLog( " ... copying: " + Packet.DestinationFile.FullName, ConsoleColor.DarkMagenta );
EnsureFolderExists( Packet.DestinationFile.Directory );
if( Packet.DestinationFile.Exists )
{
Packet.DestinationFile.IsReadOnly = false;
}
Packet.SourceFile.CopyTo( Packet.DestinationFile.FullName, true );
if( Packet.Hash.Length > 0 )
{
VerifyFilePackets.Enqueue( Packet );
}
}
catch( Exception Ex )
{
Warning( "Failed to copy: " + Packet.DestinationFile.FullName + " (" + Ex.Message + ") -- trying again!" );
Packet.SourceFile.Refresh();
Packet.DestinationFile.Refresh();
CopyFilePackets.Enqueue( Packet );
}
}
}
}
private static void VerifyFilesThreadProc()
{
while( bVerifyingFiles )
{
while( VerifyFilePackets.Count > 0 )
{
CopyFilePacket Packet = null;
try
{
bool bVerificationSuccessful = true;
Packet = VerifyFilePackets.Dequeue();
DetailLog( " ... verifying: " + Packet.DestinationFile.FullName, ConsoleColor.DarkGreen );
Packet.DestinationFile.Refresh();
if( !Packet.DestinationFile.Exists )
{
Warning( "Failed to locate for verification: " + Packet.DestinationFile.FullName );
bVerificationSuccessful = false;
}
else
{
byte[] HashData = Crypto.CreateChecksum( Packet.DestinationFile );
string NewHash = Convert.ToBase64String( HashData );
if( NewHash != Packet.Hash )
{
Warning( "Verification checksum failed: " + Packet.DestinationFile.FullName + " (" + NewHash + " != " + Packet.Hash + ")" );
bVerificationSuccessful = false;
}
}
if( !bVerificationSuccessful )
{
Packet.SourceFile.Refresh();
Packet.DestinationFile.Refresh();
CopyFilePackets.Enqueue( Packet );
}
}
catch( Exception Ex )
{
Warning( "Failed to verify: " + Packet.DestinationFile.FullName + " (" + Ex.Message + ") -- trying again!" );
VerifyFilePackets.Enqueue( Packet );
}
}
}
}
/// <summary>Copy all the files in the TOC to the destination folder(s)</summary>
/// <param name="TableOfContents">Container for all the files to copy</param>
public static void CopyFiles( TOC TableOfContents )
{
if( Options.Destination.Length > 0 )
{
long TotalFileCount = 0;
long TotalBytesCopied = 0;
bCopyingFiles = true;
Thread CopyFilesThread = new Thread( CopyFilesThreadProc );
CopyFilesThread.Start();
bVerifyingFiles = true;
Thread VerifyFilesThread = new Thread( VerifyFilesThreadProc );
VerifyFilesThread.Start();
foreach( TOCEntry Entry in TableOfContents.Entries )
{
// Handle file copies
FileInfo SourceFile = Entry.Info;
FileInfo DestFile = new FileInfo( Path.Combine( Options.Destination, Entry.Name ) );
if( CopyRequired( SourceFile, DestFile ) )
{
// Create copy packet and add to queue
CopyFilePacket Packet = new CopyFilePacket( SourceFile, DestFile, Entry.Hash );
CopyFilePackets.Enqueue( Packet );
TotalFileCount++;
TotalBytesCopied += SourceFile.Length;
}
else
{
DetailLog( " ... not copying: " + DestFile.FullName, ConsoleColor.DarkYellow );
}
}
while( CopyFilePackets.Count > 0 || VerifyFilePackets.Count > 0 )
{
if( !Options.bLog )
{
ProgressLog( " ... waiting for " + CopyFilePackets.Count + " copies, and " + VerifyFilePackets.Count + " verifies ", ConsoleColor.Cyan );
}
Thread.Sleep( 100 );
}
if( !Options.bLog )
{
ProgressLog( " ... waiting for " + CopyFilePackets.Count + " copies, and " + VerifyFilePackets.Count + " verifies ", ConsoleColor.Cyan );
}
bCopyingFiles = false;
bVerifyingFiles = false;
Log( "", ConsoleColor.Green );
Log( "Completed copying " + TotalFileCount + " files totaling " + TotalBytesCopied + " bytes", ConsoleColor.Green );
}
}
}
}

292
Engine/Source/Programs/Distill/Private/Selection.cs
View File

@@ -1,292 +0,0 @@
// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace Tools.Distill
{
public partial class Distill
{
/// <summary>Get the tagset to use to define the set of files to operate on</summary>
/// <param name="Settings">The contents of the configuration file</param>
/// <returns>The tagset that defines which files to operate on</returns>
public static TagSet GetTagSet( DistillSettings Settings )
{
TagSet Tags =
(
from TagSetDetail in Settings.TagSets
where TagSetDetail.Name == Options.TagSet
select TagSetDetail
).FirstOrDefault();
if( Tags == null )
{
Error( "Failed to find tagset: " + Options.TagSet );
return null;
}
Log( " ... using tagset " + Tags.Name + " with " + Tags.Tags.Length + " tags.", ConsoleColor.Cyan );
return Tags;
}
/// <summary>Gets the list of filesets with macros from the configuration file</summary>
/// <param name="Settings">The contents of the configuration file</param>
/// <param name="Set">The tagset to look for</param>
/// <returns>A list of filesets including macros</returns>
public static List<FileSet> GetFileSets( DistillSettings Settings, TagSet Set )
{
IEnumerable<FileSet[]> BaseSets =
(
from Group in Settings.FileGroups
where Set.Tags.Contains( Group.Tag )
where Group.Platform == null || Group.Platform.Contains( Options.Platform )
select Group.Files
);
List<FileSet> Sets = new List<FileSet>();
foreach( FileSet[] FileSet in BaseSets )
{
Sets.AddRange( FileSet );
}
return Sets;
}
private static string[] ExpandFilterMacros( string[] OldFilters )
{
if( OldFilters == null )
{
return null;
}
List<string> NewFilters = new List<string>();
foreach( string OldFilter in OldFilters )
{
string NewFilter = OldFilter;
NewFilter = NewFilter.Replace( "%GAME%", Options.Game );
NewFilter = NewFilter.Replace( "%PLATFORM%", Options.Platform );
NewFilter = NewFilter.Replace( "%LANGUAGE%", Options.Region );
if( NewFilter.Contains( "%NOTGAME%" ) )
{
NewFilters.AddRange( Options.NotGames.Select( x => NewFilter.Replace( "%NOTGAME%", x ) ) );
}
else if( NewFilter.Contains( "%NOTPLATFORM%" ) )
{
NewFilters.AddRange( Options.NotPlatforms.Select( x => NewFilter.Replace( "%NOTPLATFORM%", x ) ) );
// Always keep "Windows" folder is specified platform was a Windows platform
if( Options.Platform == "Win64" || Options.Platform == "Win32" )
{
NewFilters.Remove( "Windows" );
}
}
else if( NewFilter.Contains( "%NOTLANGUAGE%" ) )
{
NewFilters.AddRange( Options.NotLanguages.Select( x => NewFilter.Replace( "%NOTLANGUAGE%", x ) ) );
}
else
{
NewFilters.Add( NewFilter );
}
}
return NewFilters.ToArray();
}
/// <summary>Generate a new list of filesets with all macros (e.g. %GAME%) expanded to their real values</summary>
/// <param name="FileSets">List of filesets potentially containing macros</param>
/// <param name="Settings">The settings used to fill in the macros</param>
/// <returns>List of filesets with evaluated macros</returns>
public static List<FileSet> ExpandMacros( List<FileSet> FileSets, DistillSettings Settings )
{
List<FileSet> NewFileSets = new List<FileSet>();
Options.NotPlatforms = new List<string>( Settings.KnownPlatforms.Where( x => x.Name != Options.Platform && !Options.IncPlatforms.Contains(x.Name) ).Select( x => x.Name ) );
Options.NotLanguages = new List<string>( Settings.KnownLanguages.Where( x => x != Options.Region ) );
foreach( FileSet OldFileSet in FileSets )
{
FileSet NewFileSet = new FileSet();
NewFileSet.bIsRecursive = OldFileSet.bIsRecursive;
NewFileSet.Path = OldFileSet.Path;
NewFileSet.Path = NewFileSet.Path.Replace( "%GAME%", Options.Game );
NewFileSet.Path = NewFileSet.Path.Replace( "%PLATFORM%", Options.Platform );
NewFileSet.Path = NewFileSet.Path.Replace( "%LANGUAGE%", Options.Region );
NewFileSet.Path = NewFileSet.Path.Replace( "\\", "/" );
NewFileSet.FilterOutFiles = ExpandFilterMacros(OldFileSet.FilterOutFiles);
NewFileSet.FilterOutFolders = ExpandFilterMacros(OldFileSet.FilterOutFolders);
if (NewFileSet.FilterOutFolders != null)
{
List<string> NewFolders = new List<string>();
foreach (string FilterOutFolder in NewFileSet.FilterOutFolders)
{
string NewFilter = "/" + FilterOutFolder + "/";
NewFolders.Add(NewFilter);
}
NewFileSet.FilterOutFolders = NewFolders.ToArray();
}
NewFileSets.Add( NewFileSet );
}
Log( " ... found " + NewFileSets.Count + " file sets.", ConsoleColor.Cyan );
return NewFileSets;
}
private static List<FileInfo> GetFiles( string DirectoryName, string FileName, bool bIsRecursive )
{
List<FileInfo> Files = new List<FileInfo>();
DetailLog( "Checking: " + Path.Combine( DirectoryName, FileName ) + ( bIsRecursive ? " (recursively)" : "" ), ConsoleColor.DarkCyan );
DirectoryInfo DirInfo = new DirectoryInfo( DirectoryName );
if( DirInfo.Exists )
{
Files.AddRange( DirInfo.GetFiles( FileName ).ToList() );
DetailLog( " ... added " + Files.Count + " files.", ConsoleColor.DarkCyan );
if( bIsRecursive )
{
foreach( DirectoryInfo SubDirInfo in DirInfo.GetDirectories() )
{
Files.AddRange( GetFiles( SubDirInfo.FullName, FileName, true ) );
}
}
}
return Files;
}
private static List<Regex> ConvertToRegularExpressions( string[] Filters )
{
List<Regex> RegExpFilters = new List<Regex>();
if( Filters != null )
{
foreach( string Filter in Filters )
{
string Escaped = Regex.Escape( Filter );
Escaped = Escaped.Replace( "\\*", ".*" );
Escaped = Escaped.Replace( "\\?", "." );
Regex RegExp = new Regex( Escaped, RegexOptions.Compiled | RegexOptions.IgnoreCase );
RegExpFilters.Add( RegExp );
}
}
return RegExpFilters;
}
private static bool CheckFolder(FileInfo Info, List<Regex> FilterOutFolders)
{
string InfoDirectory = Info.Directory.ToString();
string RelativePath = InfoDirectory.Substring(Environment.CurrentDirectory.Length + 1);
string[] Directories = RelativePath.Split( "\\/".ToCharArray() );
bool Check =
(
from Dir in Directories
from Filter in FilterOutFolders
where Filter.IsMatch("/" + Dir + "/")
select Dir
).Any();
return !Check;
}
private static bool CheckFile(FileInfo Info, List<Regex> FilterOutFiles)
{
bool Check =
(
from Filter in FilterOutFiles
where Filter.IsMatch( Info.Name )
select Filter
).Any();
return !Check;
}
private static List<FileInfo> FilterFiles(string[] FilterOutFiles, string[] FilterOutFolders, List<FileInfo> PotentialFiles)
{
List<FileInfo> GoodFiles = new List<FileInfo>();
if (FilterOutFiles != null || FilterOutFolders != null || Options.bWhitelistSignatures)
{
List<Regex> FileRegExpFilters = ConvertToRegularExpressions(FilterOutFiles);
List<Regex> FolderRegExpFilters = ConvertToRegularExpressions(FilterOutFolders);
DetailLog( " ... using " + FileRegExpFilters.Count + " file, and " + FolderRegExpFilters.Count + " folder filters", ConsoleColor.DarkCyan );
foreach( FileInfo Info in PotentialFiles )
{
DetailLog( "Filtering: " + Info.FullName, ConsoleColor.DarkCyan );
if( !CheckFolder( Info, FolderRegExpFilters ) )
{
DetailLog( " ... filtered by folder", ConsoleColor.DarkCyan );
continue;
}
if( !CheckFile( Info, FileRegExpFilters ) )
{
DetailLog( " ... filtered by file", ConsoleColor.DarkCyan );
continue;
}
if( Options.bWhitelistSignatures )
{
if( Info.Extension.ToLower() == ".dll" || Info.Extension.ToLower() == ".exe" )
{
if( !Crypto.ValidateDigitalSignature( Info ) )
{
DetailLog( " ... filtered by signature", ConsoleColor.DarkCyan );
continue;
}
}
}
GoodFiles.Add( Info );
}
}
else
{
DetailLog( " ... unfiltered", ConsoleColor.DarkCyan );
return PotentialFiles;
}
return GoodFiles;
}
/// <summary>Find all the local files that match the criteria defined in the list of filesets</summary>
/// <param name="FileSets">A list of filesets that define the files to look for</param>
/// <returns>A list of FileInfos of all the found files</returns>
public static List<FileInfo> GetFiles( List<FileSet> FileSets )
{
List<FileInfo> Files = new List<FileInfo>();
foreach( FileSet SourceSet in FileSets )
{
string DirectoryName = Path.Combine( Options.Source, Path.GetDirectoryName( SourceSet.Path ) );
string FileName = Path.GetFileName( SourceSet.Path );
List<FileInfo> PotentialFiles = GetFiles( DirectoryName, FileName, SourceSet.bIsRecursive );
if( PotentialFiles.Count > 0 )
{
List<FileInfo> FilteredFiles = FilterFiles(SourceSet.FilterOutFiles, SourceSet.FilterOutFolders, PotentialFiles);
Files.AddRange( FilteredFiles );
}
}
Log( " ... found " + Files.Count + " files.", ConsoleColor.Cyan );
return Files;
}
}
}

142
Engine/Source/Programs/Distill/Private/Settings.cs
View File

@@ -1,142 +0,0 @@
// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Text;
using System.Xml;
using System.Xml.Serialization;
namespace Tools.Distill
{
/// <summary>A container class for the details of the platform</summary>
public class PlatformInfo
{
/// <summary>The name of the platform</summary>
[XmlAttribute]
public string Name;
/// <summary>Whether this platform has a case sensitive file system</summary>
[XmlAttribute]
[DefaultValue( false )]
public bool bCaseSensitiveFileSystem;
/// <summary>An empty constructor for XML serialisation support</summary>
public PlatformInfo()
{
bCaseSensitiveFileSystem = false;
}
/// <summary>A debugging override</summary>
/// <returns>A string of the name of the platform</returns>
public override string ToString()
{
return Name;
}
}
/// <summary>Groups of tags so that a sync can be specified with one name, and do multiple FileGroup tags</summary>
public class TagSet
{
/// <summary>The name of the tagset</summary>
[XmlAttribute]
public string Name;
/// <summary>An array of tags that make up this tagset</summary>
[XmlArray]
public string[] Tags;
/// <summary>An empty constructor for XML serialisation support</summary>
public TagSet()
{
}
/// <summary>A debugging override</summary>
/// <returns>A string of the name of the tagset</returns>
public override string ToString()
{
return Name;
}
}
/// <summary>A set of files to include</summary>
public class FileSet
{
/// <summary>Wildcard that specifies some files to sync</summary>
[XmlAttribute]
public string Path;
/// <summary>Should this set of files to a recursive sync</summary>
[XmlAttribute]
[DefaultValue( false )]
public bool bIsRecursive;
/// <summary>Optional filter to apply to files (array of filenames) to not copy them</summary>
[XmlArray]
public string[] FilterOutFiles;
/// <summary>Optional filter to apply to folders (array of folder names) to not copy them</summary>
[XmlArray]
public string[] FilterOutFolders;
/// <summary>An empty constructor for XML serialisation support</summary>
public FileSet()
{
bIsRecursive = false;
}
/// <summary>A debugging override</summary>
/// <returns>A string of the fileset path</returns>
public override string ToString()
{
return Path;
}
}
/// <summary>A group of filesets</summary>
public class FileGroup
{
/// <summary>Platform for this group. Can also be * for any (same as null), or Console for non-PC platforms</summary>
[XmlAttribute]
public string[] Platform;
/// <summary>Tag for this group.</summary>
[XmlAttribute]
public string Tag;
/// <summary>List of file infos for the group</summary>
[XmlArray]
public FileSet[] Files;
/// <summary>An empty constructor for XML serialisation support</summary>
public FileGroup()
{
}
}
/// <summary>A container class for all information about which files to copy</summary>
public class DistillSettings
{
/// <summary>List of known languages (populated automatically)</summary>
[XmlIgnore]
public string[] KnownLanguages;
/// <summary>List of known platforms</summary>
[XmlArray]
public PlatformInfo[] KnownPlatforms;
/// <summary>List of TagSet objects</summary>
[XmlArray]
public TagSet[] TagSets;
/// <summary>Set of file groups for syncing</summary>
[XmlArray]
public FileGroup[] FileGroups;
/// <summary>An empty constructor for XML serialisation support</summary>
public DistillSettings()
{
}
}
}

166
Engine/Source/Programs/Distill/Private/TOC.cs
View File

@@ -1,166 +0,0 @@
// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Xml;
using System.Xml.Serialization;
namespace Tools.Distill
{
/// <summary>A container class for the information stored in the TOC about a single file</summary>
public class TOCEntry
{
/// <summary>A cached FileInfo for easy access later</summary>
[XmlIgnore]
public FileInfo Info = null;
/// <summary>The path of the file relative to the root of the branch</summary>
[XmlAttribute]
public string Name = "";
/// <summary>The size of the file in bytes</summary>
[XmlAttribute]
public long Size = 0;
/// <summary>The time the file was last written to in 100ns UTC ticks</summary>
[XmlAttribute]
public long LastWriteTimeUTC = DateTime.MinValue.Ticks;
/// <summary>The forty byte SHA256 checksum in base 64 notation (optional)</summary>
[XmlAttribute]
public string Hash = "";
/// <summary>An empty constructor for XML serialisation support</summary>
public TOCEntry()
{
}
/// <summary>An Equals override that just checks the name</summary>
/// <param name="Other">The TOCEntry to compare against</param>
/// <returns>True if the Name fields in both objects are identical</returns>
public override bool Equals( object Other )
{
TOCEntry TOCOther = ( TOCEntry )Other;
return Name == TOCOther.Name;
}
/// <summary>A GetHashCode override to suppress the warning</summary>
/// <returns>A hash code</returns>
public override int GetHashCode()
{
return base.GetHashCode();
}
/// <summary>A ToString() override to ease debugging</summary>
/// <returns>Relative path of the file</returns>
public override string ToString()
{
return Name;
}
/// <summary>A constructor that extracts all the required information from the FileInfo, and optionally generates a SHA256 hash</summary>
/// <param name="InInfo">The FileInfo of the file to create a TOCEntry for</param>
public TOCEntry( FileInfo InInfo )
{
if( InInfo.Exists )
{
Info = InInfo;
Name = Info.FullName.Substring( Distill.Options.Source.Length + 1 ).Replace( "\\", "/" );
Size = Info.Exists ? Info.Length : 0;
LastWriteTimeUTC = Info.LastWriteTimeUtc.Ticks;
if( Distill.Options.bChecksum )
{
byte[] HashData = Crypto.CreateChecksum( Info );
Hash = Convert.ToBase64String( HashData );
}
}
}
}
/// <summary>A container class for a list of TOCEntries</summary>
public class TOC
{
/// <summary>A list of table of contents entries; one per file</summary>
[XmlArray]
public List<TOCEntry> Entries = null;
/// <summary>An empty constructor for XML serialisation support</summary>
public TOC()
{
}
/// <summary>A constructor that generates the table of contents entries from a list of FileInfos</summary>
/// <param name="Files">A list of FileInfo classes containing files to create TOCEntries for</param>
public TOC( List<FileInfo> Files )
{
Entries = new List<TOCEntry>( Files.Select( x => new TOCEntry( x ) ) );
}
/// <summary>A compare function that lists the differences between two TOC files</summary>
/// <param name="Other">The TOC class to compare against</param>
/// <remarks>The files referenced in the newly created TOC are compared against the files referenced in the old TOC. All files in both TOCs have their SHA256 checksums compared,
/// then files that no longer exist are listed, then new files that did not originally exist are listed.</remarks>
public void Compare( TOC Other )
{
// Work out which files have different SHA256 hash values
List<string> HashDifferences =
(
from Entry in Entries
from OtherEntry in Other.Entries
where Entry.Name == OtherEntry.Name
where Entry.Hash != OtherEntry.Hash
select Entry.Name
).ToList();
if( HashDifferences.Count == 0 )
{
Distill.Log( "No files with mismatched hash values", ConsoleColor.Green );
}
else
{
Distill.Log( HashDifferences.Count + " file(s) with mismatched hash values", ConsoleColor.Red );
HashDifferences.ForEach( x => Distill.Log( " ... mismatched hash: " + x.ToString(), ConsoleColor.Red ) );
}
// Work out which files do not currently exist
List<string> MissingFiles =
(
from Entry in Entries
where !Other.Entries.Contains( Entry )
select Entry.Name
).ToList();
if( MissingFiles.Count == 0 )
{
Distill.Log( "No missing files", ConsoleColor.Green );
}
else
{
Distill.Log( MissingFiles.Count + " missing file(s)", ConsoleColor.Red );
MissingFiles.ForEach( x => Distill.Log( " ... missing: " + x.ToString(), ConsoleColor.Red ) );
}
// Work out which files have been created
List<string> NewFiles =
(
from Entry in Other.Entries
where !Entries.Contains( Entry )
select Entry.Name
).ToList();
if( NewFiles.Count == 0 )
{
Distill.Log( "No new files", ConsoleColor.Green );
}
else
{
Distill.Log( NewFiles.Count + " new file(s)", ConsoleColor.Red );
NewFiles.ForEach( x => Distill.Log( " ... new: " + x.ToString(), ConsoleColor.Red ) );
}
}
}
}

40
Engine/Source/Programs/Distill/Private/Zip.cs
View File

@@ -1,40 +0,0 @@
// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Ionic.Zip;
namespace Tools.Distill
{
public partial class Distill
{
/// <summary>Copy all the files in the TOC to the all the destination zips</summary>
/// <param name="TableOfContents">Container for all the files to copy</param>
public static void ZipFiles( TOC TableOfContents )
{
// Handle zips
if( Options.ZipName.Length > 0 )
{
long TotalFilesZipped = TableOfContents.Entries.Count;
long TotalBytesZipped = TableOfContents.Entries.Sum( x => x.Info.Length );
ZipFile Zip = new ZipFile( Options.ZipName );
Zip.CompressionLevel = Ionic.Zlib.CompressionLevel.Level9;
Zip.UseZip64WhenSaving = Zip64Option.Always;
Zip.BufferSize = 0x10000;
TableOfContents.Entries.ForEach( x => Zip.UpdateFile( x.Name ) );
Log( " ... saving zip: " + Zip.Name, ConsoleColor.Green );
Zip.Save();
FileInfo ZipInfo = new FileInfo( Zip.Name );
Log( "Completed saving zip with " + TotalFilesZipped + " files to " + ZipInfo.Length + " bytes (from " + TotalBytesZipped + ")", ConsoleColor.Green );
}
}
}
}

11
Engine/Source/Programs/Distill/Properties/AssemblyInfo.cs
View File

@@ -1,11 +0,0 @@
// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
using System.Reflection;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle( "Distill" )]
[assembly: AssemblyDescription( "A utility to copy, ftp, zip, and validate builds." )]
[assembly: AssemblyConfiguration( "" )]

3
Engine/Source/Programs/UnrealBuildTool/System/ProjectFileGenerator.cs
View File

@@ -511,9 +511,6 @@ namespace UnrealBuildTool
// Add automation.csproj files to the master project
AddAutomationModules(ProgramsFolder);
// Add Distill to the master project
ProgramsFolder.ChildProjects.Add(AddSimpleCSharpProject("Distill"));
// Add DotNETUtilities to the master project
ProgramsFolder.ChildProjects.Add(AddSimpleCSharpProject("DotNETCommon/DotNETUtilities", bShouldBuildForAllSolutionTargets: true, bForceDevelopmentConfiguration: true));