UnrealEngineUWP/Engine/Source/Programs/CrashReporter/CrashReportProcess/ReportQueue.cs

// Copyright 1998-2017 Epic Games, Inc. All Rights Reserved.

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using Tools.CrashReporter.CrashReportCommon;
using Tools.DotNETCommon.XmlHandler;

namespace Tools.CrashReporter.CrashReportProcess
{
	using ReportIdSet = HashSet<string>;

	interface IReportQueue : IDisposable
	{
		int CheckForNewReports();
		bool TryDequeueReport(out FGenericCrashContext Context);
		void CleanLandingZone();

		string QueueId { get; }
		string LandingZonePath { get; }
	}

	abstract class ReportQueueBase : IReportQueue
	{
		// PROPERTIES

		protected int QueueCount
		{
			get
			{
				lock (NewReportsLock)
				{
					return NewCrashContexts.Count;
				}
			}
		}

		protected int LastQueueSizeOnDisk { get; private set; }

		public string QueueId
		{
			get { return QueueName; }
		}

		public string LandingZonePath
		{
			get { return LandingZone; }
		}

		protected abstract string QueueProcessingStartedEventName { get; }

		// METHODS

		protected ReportQueueBase(string InQueueName, string LandingZonePath, int InDecimateWaitingCountStart, int InDecimateWaitingCountEnd)
		{
			QueueName = InQueueName;
			LandingZone = LandingZonePath;
			DecimateWaitingCountStart = InDecimateWaitingCountStart;
			DecimateWaitingCountEnd = InDecimateWaitingCountEnd;
			CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(LandingZonePath, Config.Default.DiskSpaceAlertPercent);
		}

		/// <summary>
		/// Look for new report folders and add them to the publicly available thread-safe queue.
		/// </summary>
		public int CheckForNewReports()
		{
			try
			{
				if (QueueCount >= Config.Default.MinDesiredMemoryQueueSize)
				{
					CrashReporterProcessServicer.WriteEvent(string.Format(
						"CheckForNewReports: {0} skipped busy queue size {1} in {2}", QueueName, QueueCount, LandingZone));
				}
				else
				{
					var NewReportPath = "";
					var ReportsInLandingZone = new ReportIdSet();

					CrashReporterProcessServicer.StatusReporter.AlertOnLowDisk(LandingZone, Config.Default.DiskSpaceAlertPercent);

					DirectoryInfo[] DirectoriesInLandingZone;
					if (GetCrashesFromLandingZone(out DirectoriesInLandingZone))
					{
						LastQueueSizeOnDisk = DirectoriesInLandingZone.Length;

						int EnqueuedCount = 0;
						CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: {0} reports in disk landing zone {1}",
							DirectoriesInLandingZone.Length, LandingZone));

						// Add any new reports
						for (int DirIndex = 0; DirIndex < DirectoriesInLandingZone.Length && QueueCount < Config.Default.MaxMemoryQueueSize; DirIndex++)
						{
							var SubDirInfo = DirectoriesInLandingZone[DirIndex];
							try
							{
								if (Directory.Exists(SubDirInfo.FullName))
								{
									NewReportPath = SubDirInfo.FullName;
									ReportsInLandingZone.Add(NewReportPath);
									if (!ReportsInLandingZoneLastTimeWeChecked.Contains(NewReportPath))
									{
										if (EnqueueNewReport(NewReportPath))
										{
											EnqueuedCount++;
										}
									}
								}
							}
							catch (Exception Ex)
							{
								CrashReporterProcessServicer.WriteException("CheckForNewReportsInner: " + Ex, Ex);
								ReportProcessor.MoveReportToInvalid(NewReportPath, "NEWRECORD_FAIL_" + DateTime.Now.Ticks);
							}
						}

						ReportsInLandingZoneLastTimeWeChecked = ReportsInLandingZone;

						CrashReporterProcessServicer.WriteEvent(string.Format(
							"CheckForNewReports: {0} enqueued to queue size {1} from {2}", EnqueuedCount, QueueCount, LandingZone));
						CrashReporterProcessServicer.WriteEvent(string.Format("CheckForNewReports: Enqueue rate {0:N1}/minute from {1}",
							EnqueueCounter.EventsPerSecond*60, LandingZone));
					}
					else
					{
						LastQueueSizeOnDisk = 0;
					}
				}
			}
			catch (Exception Ex)
			{
				CrashReporterProcessServicer.WriteException("CheckForNewReportsOuter: " + Ex, Ex);
			}

			return GetTotalWaitingCount();
		}

		/// <summary>
		/// Tidy up the landing zone folder
		/// </summary>
		public void CleanLandingZone()
		{
			var DirInfo = new DirectoryInfo(LandingZone);
			foreach (var SubDirInfo in DirInfo.GetDirectories())
			{
				if (SubDirInfo.LastWriteTimeUtc.AddDays(Config.Default.DeleteWaitingReportsDays) < DateTime.UtcNow)
				{
					SubDirInfo.Delete(true);
				}
			}
		}

		public virtual void Dispose()
		{
			// Attempt to remove the queued crashes from the ReportIndex
			lock (NewReportsLock)
			{
				CrashReporterProcessServicer.WriteEvent(string.Format("{0} shutting down", QueueName));
				CrashReporterProcessServicer.WriteEvent(string.Format("{0} dequeuing {1} crashes for next time", QueueName, NewCrashContexts.Count));

				while (NewCrashContexts.Count > 0)
				{
					try
					{
						string ReportName = Path.GetFileName(NewCrashContexts.Peek().CrashDirectory);
						CrashReporterProcessServicer.ReportIndex.TryRemoveReport(ReportName);
					}
					finally
					{
						NewCrashContexts.Dequeue();
					}
				}
			}
		}

		protected abstract int GetTotalWaitingCount();

		protected virtual bool GetCrashesFromLandingZone(out DirectoryInfo[] OutDirectories)
		{
			// Check the landing zones for new reports
			DirectoryInfo DirInfo = new DirectoryInfo(LandingZone);

			// Couldn't find a landing zone, skip and try again later.
			// Crash receiver will recreate them if needed.
			if (!DirInfo.Exists)
			{
				CrashReporterProcessServicer.WriteFailure("LandingZone not found: " + LandingZone);
				OutDirectories = new DirectoryInfo[0];
				return false;
			}

			OutDirectories = DirInfo.GetDirectories().OrderBy(dirinfo => dirinfo.CreationTimeUtc).ToArray();
			return true;
		}

		/// <summary> Looks for the CrashContext.runtime-xml, if found, will return a new instance of the FGenericCrashContext. </summary>
		private static FGenericCrashContext FindCrashContext(string NewReportPath)
		{
			string CrashContextPath = Path.Combine(NewReportPath, FGenericCrashContext.CrashContextRuntimeXMLName);
			bool bExist = File.Exists(CrashContextPath);

			if (bExist)
			{
				return FGenericCrashContext.FromFile(CrashContextPath);
			}

			return null;
		}

		/// <summary> Converts WER metadata xml file to the crash context. </summary>
		private static void ConvertMetadataToCrashContext(FReportData ReportData, string NewReportPath, ref FGenericCrashContext OutCrashContext)
		{
			if (OutCrashContext == null)
			{
				OutCrashContext = new FGenericCrashContext();
			}

			OutCrashContext.PrimaryCrashProperties.CrashVersion = (int)ECrashDescVersions.VER_1_NewCrashFormat;

			//OutCrashContext.PrimaryCrashProperties.ProcessId = 0; don't overwrite valid ids, zero is default anyway
			OutCrashContext.PrimaryCrashProperties.CrashGUID = new DirectoryInfo(NewReportPath).Name;
			// 			OutCrashContext.PrimaryCrashProperties.IsInternalBuild
			// 			OutCrashContext.PrimaryCrashProperties.IsPerforceBuild
			// 			OutCrashContext.PrimaryCrashProperties.IsSourceDistribution
			// 			OutCrashContext.PrimaryCrashProperties.SecondsSinceStart
			OutCrashContext.PrimaryCrashProperties.GameName = ReportData.GameName;
			// 			OutCrashContext.PrimaryCrashProperties.ExecutableName
			// 			OutCrashContext.PrimaryCrashProperties.BuildConfiguration
			// 			OutCrashContext.PrimaryCrashProperties.PlatformName
			// 			OutCrashContext.PrimaryCrashProperties.PlatformNameIni
			OutCrashContext.PrimaryCrashProperties.PlatformFullName = ReportData.Platform;
			OutCrashContext.PrimaryCrashProperties.EngineMode = ReportData.EngineMode;
			OutCrashContext.PrimaryCrashProperties.EngineVersion = ReportData.GetEngineVersion();
			OutCrashContext.PrimaryCrashProperties.BuildVersion = ReportData.BuildVersion;
			OutCrashContext.PrimaryCrashProperties.CommandLine = ReportData.CommandLine;
			// 			OutCrashContext.PrimaryCrashProperties.LanguageLCID

			// Create a locate and get the language.
			int LanguageCode = 0;
			int.TryParse(ReportData.Language, out LanguageCode);
			try
			{
				if (LanguageCode > 0)
				{
					CultureInfo LanguageCI = new CultureInfo(LanguageCode);
					OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = LanguageCI.Name;
				}
			}
			catch (Exception)
			{
				OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = null;
			}

			if (string.IsNullOrEmpty(OutCrashContext.PrimaryCrashProperties.AppDefaultLocale))
			{
				// Default to en-US
				OutCrashContext.PrimaryCrashProperties.AppDefaultLocale = "en-US";
			}

			// 			OutCrashContext.PrimaryCrashProperties.IsUE4Release
			string WERUserName = ReportData.UserName;
			if (!string.IsNullOrEmpty(WERUserName) || string.IsNullOrEmpty(OutCrashContext.PrimaryCrashProperties.UserName))
			{
				OutCrashContext.PrimaryCrashProperties.UserName = WERUserName;
			}

			OutCrashContext.PrimaryCrashProperties.BaseDir = ReportData.BaseDir;
			// 			OutCrashContext.PrimaryCrashProperties.RootDir
			OutCrashContext.PrimaryCrashProperties.MachineId = ReportData.MachineId;
			OutCrashContext.PrimaryCrashProperties.LoginId = ReportData.LoginId;
			if (string.IsNullOrWhiteSpace(OutCrashContext.PrimaryCrashProperties.MachineId))
			{
				// Set MachineId from LoginId if the metadata is too new to contain MachineId
				OutCrashContext.PrimaryCrashProperties.MachineId = OutCrashContext.PrimaryCrashProperties.LoginId;
			}
			OutCrashContext.PrimaryCrashProperties.EpicAccountId = ReportData.EpicAccountId;
			// 			OutCrashContext.PrimaryCrashProperties.CallStack
			// 			OutCrashContext.PrimaryCrashProperties.SourceContext
			OutCrashContext.PrimaryCrashProperties.UserDescription = string.Join("\n", ReportData.UserDescription);

			// 			OutCrashContext.PrimaryCrashProperties.CrashDumpMode
			// 			OutCrashContext.PrimaryCrashProperties.Misc.NumberOfCores
			// 			OutCrashContext.PrimaryCrashProperties.Misc.NumberOfCoresIncludingHyperthreads
			// 			OutCrashContext.PrimaryCrashProperties.Misc.Is64bitOperatingSystem
			// 			OutCrashContext.PrimaryCrashProperties.Misc.CPUVendor
			// 			OutCrashContext.PrimaryCrashProperties.Misc.CPUBrand
			// 			OutCrashContext.PrimaryCrashProperties.Misc.PrimaryGPUBrand
			// 			OutCrashContext.PrimaryCrashProperties.Misc.OSVersionMajor
			// 			OutCrashContext.PrimaryCrashProperties.Misc.OSVersionMinor
			// 			OutCrashContext.PrimaryCrashProperties.Misc.AppDiskTotalNumberOfBytes
			// 			OutCrashContext.PrimaryCrashProperties.Misc.AppDiskNumberOfFreeBytes
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysical
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalVirtual
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.PageSize
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.TotalPhysicalGB
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.AvailablePhysical
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.AvailableVirtual
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.UsedPhysical
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedPhysical
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.UsedVirtual
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.PeakUsedVirtual
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.bIsOOM
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationSize
			// 			OutCrashContext.PrimaryCrashProperties.MemoryStats.OOMAllocationAlignment
			OutCrashContext.PrimaryCrashProperties.TimeofCrash = new DateTime(ReportData.Ticks);
			OutCrashContext.PrimaryCrashProperties.bAllowToBeContacted = ReportData.AllowToBeContacted;

			OutCrashContext.PrimaryCrashProperties.DeploymentName = ReportData.DeploymentName;
			OutCrashContext.PrimaryCrashProperties.IsEnsure = ReportData.IsEnsure;
			OutCrashContext.PrimaryCrashProperties.IsAssert = ReportData.IsAssert;
			OutCrashContext.PrimaryCrashProperties.CrashType = ReportData.CrashType;

			GetErrorMessageFromMetadata(ReportData, OutCrashContext);
		}

		private static void GetErrorMessageFromMetadata(FReportData ReportData, FGenericCrashContext CrashContext)
		{
			if (string.IsNullOrEmpty(CrashContext.PrimaryCrashProperties.ErrorMessage))
			{
				CrashContext.PrimaryCrashProperties.ErrorMessage = string.Join("\n", ReportData.ErrorMessage);
			}
		}

		/// <summary> Looks for the WER metadata xml file, if found, will return a new instance of the WERReportMetadata. </summary>
		private static WERReportMetadata FindMetadata(string NewReportPath)
		{
			WERReportMetadata MetaData = null;

			// Just to verify that the report is still there.
			DirectoryInfo DirInfo = new DirectoryInfo(NewReportPath);
			if (!DirInfo.Exists)
			{
				CrashReporterProcessServicer.WriteFailure("FindMetadata: Directory not found " + NewReportPath);
			}
			else
			{
				// Check to see if we wish to suppress processing of this report.	
				foreach (var Info in DirInfo.GetFiles("*.xml"))
				{
					var MetaDataToCheck = XmlHandler.ReadXml<WERReportMetadata>(Info.FullName);
					if (CheckMetaData(MetaDataToCheck))
					{
						MetaData = MetaDataToCheck;
						break;
					}
				}
			}

			return MetaData;
		}

		/// <summary> Looks for the Diagnostics.txt file and returns true if exists. </summary>
		private static bool FindDiagnostics(string NewReportPath)
		{
			bool bFound = false;

			DirectoryInfo DirInfo = new DirectoryInfo(NewReportPath);
			foreach (var Info in DirInfo.GetFiles(CrashReporterConstants.DiagnosticsFileName))
			{
				bFound = true;
				break;
			}
			return bFound;
		}

		/// <summary>
		/// Optionally don't process some reports based on the Windows Error report meta data.
		/// </summary>
		/// <param name="WERData">The Windows Error Report meta data.</param>
		/// <returns>false to reject the report.</returns>
		private static bool CheckMetaData(WERReportMetadata WERData)
		{
			if (WERData == null)
			{
				return false;
			}

			// Reject any crashes with the invalid metadata.
			if (WERData.ProblemSignatures == null || WERData.DynamicSignatures == null || WERData.OSVersionInformation == null)
			{
				return false;
			}

			// Reject any crashes from the minidump processor.
			if (WERData.ProblemSignatures.Parameter0 != null && WERData.ProblemSignatures.Parameter0.ToLower() == "MinidumpDiagnostics".ToLower())
			{
				return false;
			}

			return true;
		}

		// From CrashUpload.cpp
		/*
		struct FCompressedCrashFile : FNoncopyable
		{
			int32 CurrentFileIndex; // 4 bytes for file index
			FString Filename; // 4 bytes for length + 260 bytes for char data
			TArray<uint8> Filedata; // 4 bytes for length + N bytes for data
		}
		*/

		/// <summary> Enqueues a new crash. </summary>
		private bool EnqueueNewReport(string NewReportPath)
		{
			string ReportName = Path.GetFileName(NewReportPath);

			string CompressedReportPath = Path.Combine(NewReportPath, ReportName + ".ue4crash");
			string MetadataPath = Path.Combine(NewReportPath, ReportName + ".xml");
			bool bIsCompressed = File.Exists(CompressedReportPath) && File.Exists(MetadataPath);
			if (bIsCompressed)
			{
				FCompressedCrashInformation CompressedCrashInformation = XmlHandler.ReadXml<FCompressedCrashInformation>(MetadataPath);
				bool bResult = DecompressReport(CompressedReportPath, CompressedCrashInformation);
				if (!bResult)
				{
					ReportProcessor.MoveReportToInvalid(NewReportPath, "DECOMPRESSION_FAIL_" + DateTime.Now.Ticks + "_" + ReportName);
					return false;
				}
				else
				{
					// Rename metadata file to not interfere with the WERReportMetadata.
					File.Move(MetadataPath, Path.ChangeExtension(MetadataPath, "processed_xml"));
				}
			}

			// Unified crash reporting
			FGenericCrashContext GenericContext = FindCrashContext(NewReportPath);
			FGenericCrashContext Context = GenericContext;

			bool bContextDirty = false;
			WERReportMetadata MetaData = FindMetadata(NewReportPath);
			if (MetaData != null)
			{
				if (Context == null)
				{
					// Missing crash context
					FReportData ReportData = new FReportData(MetaData, NewReportPath);
					ConvertMetadataToCrashContext(ReportData, NewReportPath, ref Context);
					bContextDirty = true;
				}
				else if (!Context.HasProcessedData())
				{
					// Missing data - try to get from WER metadata
					FReportData ReportData = new FReportData(MetaData, NewReportPath);
					GetErrorMessageFromMetadata(ReportData, Context);
					bContextDirty = true;
				}
			}

			if (Context == null)
			{
				CrashReporterProcessServicer.WriteFailure("! NoCntx  : Path=" + NewReportPath);
				ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
				return false;
			}

			Context.CrashDirectory = NewReportPath;
			Context.PrimaryCrashProperties.SetPlatformFullName();

			// Added data from WER, save to the crash context file.
			if (bContextDirty)
			{
				Context.ToFile();
			}

			FEngineVersion EngineVersion = new FEngineVersion(Context.PrimaryCrashProperties.EngineVersion);

			uint BuiltFromCL = EngineVersion.Changelist;

			string BranchName = EngineVersion.Branch;
			if (string.IsNullOrEmpty(BranchName))
			{
				CrashReporterProcessServicer.WriteEvent("% Warning NoBranch: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath + " EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion);
				Context.PrimaryCrashProperties.ProcessorFailedMessage = "Engine version has no branch name. EngineVersion=" + Context.PrimaryCrashProperties.EngineVersion;
                Context.ToFile();
            }
			else if (BranchName.Equals(CrashReporterConstants.LicenseeBranchName, StringComparison.InvariantCultureIgnoreCase))
			{
				CrashReporterProcessServicer.WriteEvent("% Warning branch is UE4-QA  : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
				Context.PrimaryCrashProperties.ProcessorFailedMessage = "Branch was the forbidden LicenseeBranchName=" + BranchName;
                Context.ToFile();
            }

			// Look for the Diagnostics.txt, if we have a diagnostics file we can continue event if the CL is marked as broken.
			bool bHasDiagnostics = FindDiagnostics(NewReportPath);

			if (BuiltFromCL == 0 && (!bHasDiagnostics && !Context.HasProcessedData()))
			{
				// For now ignore all locally made crashes.
				CrashReporterProcessServicer.WriteEvent("% Warning CL=0 and no useful data : BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
				Context.PrimaryCrashProperties.ProcessorFailedMessage = "Report from CL=0 has no diagnostics, callstack or error msg";
                Context.ToFile();
            }

			// Check static reports index for duplicated reports
			// This WILL happen when running multiple, non-mutually exclusive crash sources (e.g. Receiver + Data Router)
			// This can be safely discontinued when all crashes come from the same SQS
			// This DOES NOT scale to multiple CRP instances
			// ReportIndex can be disabled by leaving the path empty in config
			if (!CrashReporterProcessServicer.ReportIndex.TryAddNewReport(ReportName))
			{
				// Crash report not accepted by index
				CrashReporterProcessServicer.WriteEvent(string.Format(
					"EnqueueNewReport: Duplicate report skipped {0} in queue {1}", NewReportPath, QueueName));
				CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.DuplicateRejected);
				ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
				return false;
			}

			if (ShouldDecimateNextReport())
			{
				CrashReporterProcessServicer.WriteEvent(string.Format("EnqueueNewReport: Discarding Report due to backlog of {0} in queue {1}: Path={2}", GetTotalWaitingCount(), QueueName, NewReportPath));
				CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.ReportDiscarded);
				ReportProcessor.CleanReport(new DirectoryInfo(NewReportPath));
				return false;
			}

			lock (NewReportsLock)
			{
				NewCrashContexts.Enqueue(Context);
			}
			EnqueueCounter.AddEvent();
			CrashReporterProcessServicer.StatusReporter.IncrementCount(StatusReportingEventNames.QueuedEvent);
			CrashReporterProcessServicer.WriteEvent("+ Enqueued: BuiltFromCL=" + string.Format("{0,7}", BuiltFromCL) + " Path=" + NewReportPath);
			return true;
		}

		/// <summary> Tries to dequeue a report from the list. </summary>
		public bool TryDequeueReport(out FGenericCrashContext Context)
		{
			lock (NewReportsLock)
			{
				if (NewCrashContexts.Count > 0)
				{
					Context = NewCrashContexts.Dequeue();
					DequeueCounter.AddEvent();
					CrashReporterProcessServicer.StatusReporter.IncrementCount(QueueProcessingStartedEventName);
					CrashReporterProcessServicer.WriteEvent(string.Format("- Dequeued: {0:N1}/minute BuiltFromCL={1,7} Path={2}", DequeueCounter.EventsPerSecond * 60, Context.PrimaryCrashProperties.EngineVersion, Context.CrashDirectory));
					return true;
				}
			}

			Context = null;
			return false;
		}

		/// <summary> 
		/// Decompresses a compressed crash report. 
		/// </summary>
		static protected bool DecompressReport(string CompressedReportPath, FCompressedCrashInformation Meta)
		{
			string ReportPath = Path.GetDirectoryName(CompressedReportPath);
			using (FileStream FileStream = File.OpenRead(CompressedReportPath))
			{
				Int32 UncompressedSize = Meta.GetUncompressedSize();
				Int32 CompressedSize = Meta.GetCompressedSize();

				if (FileStream.Length != CompressedSize)
				{
					return false;
				}

				byte[] UncompressedBufferArray = new byte[UncompressedSize];
				byte[] CompressedBufferArray = new byte[CompressedSize];

				FileStream.Read(CompressedBufferArray, 0, CompressedSize);

				int UncompressResult = NativeMethods.UncompressMemoryZlib(UncompressedBufferArray, CompressedBufferArray);
				if (UncompressResult < 0)
				{
					CrashReporterProcessServicer.WriteFailure("! DecompressReport() failed in UncompressMemoryZlib() with " + NativeMethods.GetZlibError(UncompressResult));
					CrashReporterProcessServicer.WriteFailure("! ZLibFail in DecompressReport(): Path=" + ReportPath);
					return false;
				}

				using (BinaryReader BinaryData = new BinaryReader(new MemoryStream(UncompressedBufferArray, false)))
				{
					for (int FileIndex = 0; FileIndex < Meta.GetNumberOfFiles(); FileIndex++)
					{
						if (!WriteIncomingFile(BinaryData, FileIndex, ReportPath))
						{
							CrashReporterProcessServicer.WriteFailure("! DecompressReport() failed to write file index " + FileIndex + " Path=" + ReportPath);
							return false;
						}
					}
				}
			}

			return true;
		}

		protected static bool WriteIncomingFile(BinaryReader BinaryData, int FileIndex, string DestinationFolderPath)
		{
			try
			{
				Int32 CurrentFileIndex = BinaryData.ReadInt32();
				if (CurrentFileIndex != FileIndex)
				{
					CrashReporterProcessServicer.WriteFailure("! WriteIncomingFile index mismatch: Required=" + FileIndex + " Read=" + CurrentFileIndex);
					return false;
				}

				string Filename = FBinaryReaderHelper.ReadFixedSizeString(BinaryData);
				string SafeFilename = GetSafeFilename(Filename);
				Int32 FiledataLength = BinaryData.ReadInt32();
				byte[] Filedata = BinaryData.ReadBytes(FiledataLength);

				Directory.CreateDirectory(DestinationFolderPath);
				File.WriteAllBytes(Path.Combine(DestinationFolderPath, SafeFilename), Filedata);
				return true;
			}
			catch (Exception Ex)
			{
				throw new CrashReporterException(string.Format("! WriteIncomingFile failed writing FileIndex={0} FolderPath={1}", FileIndex, DestinationFolderPath), Ex);
			}
		}

		private bool ShouldDecimateNextReport()
		{
			bool bShouldDecimate;

			int TotalWaiting = GetTotalWaitingCount();

			if (TotalWaiting < DecimateWaitingCountStart)
			{
				// Under lower threshold - keep all crashes
				bShouldDecimate = false;
			}
			else if (TotalWaiting >= DecimateWaitingCountEnd)
			{
				// Over upper threshold - discard all crashes
				bShouldDecimate = true;
			}
			else
			{
				// Between the upper and lower queue sizes we will keep crashes in proportion with the queue size
				// e.g. Half way between the upper and lower limit - we will keep 50% of the crashes

				float Range = DecimateWaitingCountEnd - DecimateWaitingCountStart;
				float Value = TotalWaiting - DecimateWaitingCountStart;

				DecimationCounter += Value / Range;

				if (DecimationCounter > 1.0f)
				{
					DecimationCounter -= 1.0f;
					bShouldDecimate = true;
				}
				else
				{
					bShouldDecimate = false;
				}
			}

			if (bShouldDecimate)
			{
				CrashReporterProcessServicer.StatusReporter.Alert("ShouldDecimateNextReport - " + QueueName,
				                                                  "Large backlog in " + QueueName + " queue is causing crashes to be discarded.",
				                                                  Config.Default.SlackDecimateAlertRepeatMinimumMinutes);
			}

			return bShouldDecimate;
		}


		public static string GetSafeFilename(string UnsafeName)
        {
            return string.Join("X", UnsafeName.Split(Path.GetInvalidFileNameChars()));
        }

        // FIELDS

        /// <summary> A list of freshly landed crash reports. </summary>
        private readonly Queue<FGenericCrashContext> NewCrashContexts = new Queue<FGenericCrashContext>();

		/// <summary> Object used to synchronize the access to the NewReport. </summary>
		private readonly Object NewReportsLock = new Object();

		private readonly EventCounter DequeueCounter = new EventCounter(TimeSpan.FromMinutes(10), 10);

		/// <summary>Friendly queue name</summary>
		private readonly string QueueName;

		/// <summary>Incoming report path</summary>
		protected readonly string LandingZone;

		/// <summary>Report IDs that were in the folder on the last pass</summary>
		private ReportIdSet ReportsInLandingZoneLastTimeWeChecked = new ReportIdSet();

		private readonly EventCounter EnqueueCounter = new EventCounter(TimeSpan.FromMinutes(90), 20);

		/// <summary>
		/// The number of waiting reports needed to trigger decimation (discarding a fraction of the incoming reports to stop runaway backlog)
		/// </summary>
		private readonly int DecimateWaitingCountStart = Int32.MaxValue;

		/// <summary>
		/// The number of waiting reports needed to cause all reports to be discarded
		/// </summary>
		private readonly int DecimateWaitingCountEnd = Int32.MaxValue;

		private float DecimationCounter;
	}
}