// Copyright 1998-2017 Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.ServiceProcess; using Amazon.Runtime; using Amazon.S3; using Amazon.SQS; using Tools.CrashReporter.CrashReportCommon; namespace Tools.CrashReporter.CrashReportProcess { /// /// A class to handle processing received crash reports for displaying on the website. /// partial class CrashReporterProcessServicer : ServiceBase { /// A class the handle detection of new reports. public ReportWatcher Watcher = null; /// A class to lazily process reports as they are detected. public readonly List Processors = new List(); /// Current log file to write debug progress info to public static LogWriter Log = null; private static SlackWriter Slack = null; public static AmazonClient DataRouterAWS { get; private set; } public static AmazonClient OutputAWS { get; private set; } public static StatusReporting StatusReporter { get; private set; } public static Symbolicator Symbolicator { get; private set; } public static ReportIndex ReportIndex { get; private set; } /// Folder in which to store log files private static string LogFolder = null; private bool bDisposing = false; private bool bDisposed = false; /// Folder in which to store symbolication log files static public string SymbolicatorLogFolder { get; private set; } /// /// Write a status update to the event log. /// static public void WriteEvent( string Message ) { if( Message != null && Message.Length > 2 ) { Log.Print( "[STATUS] " + Message ); } } /// /// Write a status update to the event log, from the MinidumpDiagnostics. /// static public void WriteMDD( string Message ) { if( Message != null && Message.Length > 2 ) { Log.Print( "[MDD ] " + Message ); } } /// /// Write a status update to the event log, from the P4. /// static public void WriteP4( string Message ) { if( Message != null && Message.Length > 2 ) { Log.Print( "[P4 ] " + Message ); } } /// /// Write a failure to the event log. /// static public void WriteFailure( string Message ) { if( Message != null && Message.Length > 2 ) { Log.Print( "[FAILED] " + Message ); } } /// /// Write an exception message to the event log. /// static public void WriteException( string Message, Exception Ex ) { if (Ex != null) { if (Ex is OutOfMemoryException) { StatusReporter.Alert("OOM", "Out of memory.", 2); } } if (Message != null && Message.Length > 2) { Log.Print("[EXCEPT] " + Message); } else { Log.Print("[EXCEPT] NO MESSAGE"); } StatusReporter.IncrementCount(StatusReportingEventNames.ExceptionEvent); } /// /// Write to Slack. /// static public void WriteSlack(string Message) { if (Message != null && Message.Length > 0) { Slack.Write(Message); } } /// /// Initialise all the components, and create an event log. /// public CrashReporterProcessServicer() { InitializeComponent(); var StartupPath = Path.GetDirectoryName( Assembly.GetExecutingAssembly().Location ); LogFolder = Path.Combine(StartupPath, "Logs"); SymbolicatorLogFolder = Path.Combine(StartupPath, "MDDLogs"); } /// /// Start the service, and stop the service if there were any errors found. /// /// Command line arguments (unused). protected override void OnStart( string[] Arguments ) { // Create a log file for any start-up messages Log = new LogWriter("CrashReportProcess", LogFolder); Config.LoadConfig(); Slack = new SlackWriter { WebhookUrl = Config.Default.SlackWebhookUrl, Channel = Config.Default.SlackChannel, Username = Config.Default.SlackUsername, IconEmoji = Config.Default.SlackEmoji }; Symbolicator = new Symbolicator(); StatusReporter = new StatusReporting(); ReportIndex = new ReportIndex { IsEnabled = !string.IsNullOrWhiteSpace(Config.Default.ProcessedReportsIndexPath), Filepath = Config.Default.ProcessedReportsIndexPath, Retention = TimeSpan.FromDays(Config.Default.ReportsIndexRetentionDays) }; ReportIndex.ReadFromFile(); WriteEvent("Initializing AWS"); string AWSError; AWSCredentials AWSCredentialsForDataRouter = new StoredProfileAWSCredentials(Config.Default.AWSProfileInputName, Config.Default.AWSCredentialsFilepath); AmazonSQSConfig SqsConfigForDataRouter = new AmazonSQSConfig { ServiceURL = Config.Default.AWSSQSServiceInputURL }; AmazonS3Config S3ConfigForDataRouter = new AmazonS3Config { ServiceURL = Config.Default.AWSS3ServiceInputURL }; DataRouterAWS = new AmazonClient(AWSCredentialsForDataRouter, SqsConfigForDataRouter, S3ConfigForDataRouter, out AWSError); if (!DataRouterAWS.IsSQSValid || !DataRouterAWS.IsS3Valid) { WriteFailure("AWS failed to initialize profile for DataRouter access. Error:" + AWSError); StatusReporter.Alert("AWSFailInput", "AWS failed to initialize profile for DataRouter access", 0); } AWSCredentials AWSCredentialsForOutput = new StoredProfileAWSCredentials(Config.Default.AWSProfileOutputName, Config.Default.AWSCredentialsFilepath); AmazonS3Config S3ConfigForOutput = new AmazonS3Config { ServiceURL = Config.Default.AWSS3ServiceOutputURL }; OutputAWS = new AmazonClient(AWSCredentialsForOutput, null, S3ConfigForOutput, out AWSError); if (!OutputAWS.IsS3Valid) { WriteFailure("AWS failed to initialize profile for output S3 bucket access. Error:" + AWSError); StatusReporter.Alert("AWSFailOutput", "AWS failed to initialize profile for output S3 bucket access", 0); } // Add directory watchers WriteEvent("Creating ReportWatcher"); Watcher = new ReportWatcher(); WriteEvent("Creating ReportProcessors"); for (int ProcessorIndex = 0; ProcessorIndex < Config.Default.ProcessorThreadCount; ProcessorIndex++) { var Processor = new ReportProcessor(Watcher, ProcessorIndex); Processors.Add(Processor); } // Init events by enumerating event names WriteEvent("Initializing Event Counters"); FieldInfo[] EventNameFields = typeof(StatusReportingEventNames).GetFields(BindingFlags.Static | BindingFlags.Public); StatusReporter.InitCounters(EventNameFields.Select(EventNameField => (string)EventNameField.GetValue(null))); WriteEvent("Initializing Performance Mean Counters"); FieldInfo[] MeanNameFields = typeof(StatusReportingPerfMeanNames).GetFields(BindingFlags.Static | BindingFlags.Public); StatusReporter.InitMeanCounters(MeanNameFields.Select(MeanNameField => (string)MeanNameField.GetValue(null))); WriteEvent("Initializing Folder Monitors"); Dictionary FoldersToMonitor = new Dictionary(); FoldersToMonitor.Add(Config.Default.ProcessedReports, "Processed Reports"); FoldersToMonitor.Add(Config.Default.ProcessedVideos, "Processed Videos"); FoldersToMonitor.Add(Config.Default.DepotRoot, "P4 Workspace"); FoldersToMonitor.Add(Config.Default.InternalLandingZone, "CRR Landing Zone"); FoldersToMonitor.Add(Config.Default.DataRouterLandingZone, "Data Router Landing Zone"); FoldersToMonitor.Add(Config.Default.PS4LandingZone, "PS4 Landing Zone"); FoldersToMonitor.Add(Assembly.GetExecutingAssembly().Location, "CRP Binaries and Logs"); FoldersToMonitor.Add(Config.Default.MDDPDBCachePath, "MDD PDB Cache"); StatusReporter.InitFolderMonitors(FoldersToMonitor); WriteEvent("Starting StatusReporter"); StatusReporter.Start(); // Start the threads now Watcher.Start(); foreach (var Processor in Processors) { Processor.Start(); } DateTime StartupTime = DateTime.UtcNow; WriteEvent("Successfully started at " + StartupTime); } /// /// Stop the service. /// protected override void OnStop() { StatusReporter.OnPreStopping(); OnDispose(); } private void OnDispose() { bDisposing = true; // Clean up the directory watcher and crash processor threads WriteEvent("Shutdown: Stopping ReportProcessors..."); foreach (var Processor in Processors) { Processor.RequestStop(); } WriteEvent("Shutdown: Disposing ReportProcessors..."); foreach (var Processor in Processors) { Processor.Dispose(); } Processors.Clear(); WriteEvent("Shutdown: ReportProcessors stopped and disposed."); WriteEvent("Shutdown: Disposing ReportWatcher..."); Watcher.Dispose(); Watcher = null; WriteEvent("Shutdown: ReportWatcher disposed."); WriteEvent("Shutdown: Writing ReportIndex."); ReportIndex.WriteToFile(); ReportIndex = null; WriteEvent("Shutdown: ReportIndex written."); WriteEvent("Shutdown: Disposing AmazonClients..."); OutputAWS.Dispose(); OutputAWS = null; DataRouterAWS.Dispose(); DataRouterAWS = null; WriteEvent("Shutdown: AmazonClients disposed."); WriteEvent("Shutdown: Disposing StatusReporter..."); StatusReporter.Dispose(); StatusReporter = null; WriteEvent("Shutdown: StatusReporter disposed."); WriteEvent("Shutdown: Disposing SlackWriter..."); Slack.Dispose(); Slack = null; WriteEvent("Shutdown: SlackWriter disposed."); // Flush the log to disk WriteEvent("Shutdown: Disposing LogWriter."); Log.Dispose(); Log = null; bDisposed = true; } /// /// Run the service in debug mode. This spews all logging to the console rather than suppressing it. /// public void DebugRun() { OnStart( null ); Console.WriteLine( "Press enter to exit" ); Console.Read(); OnStop(); } } }