// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Text; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; namespace EpicGames.Core { /// /// Confidence of a matched log event being the correct derivation /// public enum LogEventPriority { None, Lowest, Low, BelowNormal, Normal, AboveNormal, High, Highest, } public class LogEventMatch { public LogEventPriority Priority { get; } public List Events { get; } public LogEventMatch(LogEventPriority priority, LogEvent logEvent) { Priority = priority; Events = new List { logEvent }; } public LogEventMatch(LogEventPriority priority, IEnumerable events) { Priority = priority; Events = events.ToList(); } } /// /// Interface for a class which matches error strings /// public interface ILogEventMatcher { /// /// Attempt to match events from the given input buffer /// /// The input buffer /// Information about the error that was matched, or null if an error was not matched LogEventMatch? Match(ILogCursor cursor); } /// /// Turns raw text output into structured logging events /// public class LogEventParser : IDisposable { /// /// List of event matchers for this parser /// public List Matchers { get; } = new List(); /// /// List of patterns to ignore /// public List IgnorePatterns { get; } = new List(); /// /// Buffer of input lines /// readonly LogBuffer _buffer; /// /// Buffer for holding partial line data /// readonly MemoryStream _partialLine = new MemoryStream(); /// /// Whether matching is currently enabled /// int _matchingEnabled; /// /// The inner logger /// ILogger _logger; /// /// Public accessor for the logger /// public ILogger Logger { get => _logger; set => _logger = value; } /// /// Constructor /// /// The logger to receive parsed output messages public LogEventParser(ILogger logger) { _logger = logger; _buffer = new LogBuffer(50); } /// public void Dispose() => Flush(); /// /// Enumerate all the types that implement in the given assembly, and create instances of them /// /// The assembly to enumerate matchers from public void AddMatchersFromAssembly(Assembly assembly) { foreach (Type type in assembly.GetTypes()) { if (type.IsClass && typeof(ILogEventMatcher).IsAssignableFrom(type)) { _logger.LogDebug("Adding event matcher: {Type}", type.Name); ILogEventMatcher matcher = (ILogEventMatcher)Activator.CreateInstance(type)!; Matchers.Add(matcher); } } } /// /// Writes a line to the event filter /// /// The line to output public void WriteLine(string line) { if (line.Length > 0 && line[0] == '{') { byte[] data = Encoding.UTF8.GetBytes(line); try { JsonLogEvent jsonEvent; if (JsonLogEvent.TryParse(data, out jsonEvent)) { ProcessData(true); _logger.Log(jsonEvent.Level, jsonEvent.EventId, jsonEvent, null, JsonLogEvent.Format); return; } } catch (Exception ex) { _logger.LogError(ex, "Exception while parsing log event"); } } _buffer.AddLine(StringUtils.ParseEscapeCodes(line)); ProcessData(false); } /// /// Writes data to the log parser /// /// Data to write public void WriteData(ReadOnlyMemory data) { int baseIdx = 0; int scanIdx = 0; ReadOnlySpan span = data.Span; // Handle a partially existing line if (_partialLine.Length > 0) { for (; scanIdx < span.Length; scanIdx++) { if (span[scanIdx] == '\n') { _partialLine.Write(span.Slice(baseIdx, scanIdx - baseIdx)); FlushPartialLine(); baseIdx = ++scanIdx; break; } } } // Handle any complete lines for (; scanIdx < span.Length; scanIdx++) { if(span[scanIdx] == '\n') { AddLine(data.Slice(baseIdx, scanIdx - baseIdx)); baseIdx = scanIdx + 1; } } // Add the rest of the text to the partial line buffer _partialLine.Write(span.Slice(baseIdx)); // Process the new data ProcessData(false); } /// /// Flushes the current contents of the parser /// public void Flush() { // If there's a partially written line, write that out first if (_partialLine.Length > 0) { FlushPartialLine(); } // Process any remaining data ProcessData(true); } /// /// Adds a raw utf-8 string to the buffer /// /// The string data private void AddLine(ReadOnlyMemory data) { if (data.Length > 0 && data.Span[data.Length - 1] == '\r') { data = data.Slice(0, data.Length - 1); } if (data.Length > 0 && data.Span[0] == '{') { JsonLogEvent jsonEvent; if (JsonLogEvent.TryParse(data, out jsonEvent)) { ProcessData(true); _logger.Log(jsonEvent.Level, jsonEvent.EventId, jsonEvent, null, JsonLogEvent.Format); return; } } _buffer.AddLine(StringUtils.ParseEscapeCodes(Encoding.UTF8.GetString(data.Span))); } /// /// Writes the current partial line data, with the given data appended to it, then clear the buffer /// private void FlushPartialLine() { AddLine(_partialLine.ToArray()); _partialLine.Position = 0; _partialLine.SetLength(0); } /// /// Process any data in the buffer /// /// Whether we've reached the end of the stream void ProcessData(bool bFlush) { while (_buffer.Length > 0) { // Try to match an event List? events = null; if (Regex.IsMatch(_buffer[0], "<-- Suspend Log Parsing -->", RegexOptions.IgnoreCase)) { _matchingEnabled--; } else if (Regex.IsMatch(_buffer[0], "<-- Resume Log Parsing -->", RegexOptions.IgnoreCase)) { _matchingEnabled++; } else if (_matchingEnabled >= 0) { events = MatchEvent(); } // Bail out if we need more data if (_buffer.Length < 1024 && !bFlush && _buffer.NeedMoreData) { break; } // If we did match something, check if it's not negated by an ignore pattern. We typically have relatively few errors and many more ignore patterns than matchers, so it's quicker // to check them in response to an identified error than to treat them as matchers of their own. if (events != null) { foreach (Regex ignorePattern in IgnorePatterns) { if (ignorePattern.IsMatch(_buffer[0])) { events = null; break; } } } // Report the error to the listeners if (events != null) { WriteEvents(events); _buffer.Advance(events.Count); } else { _logger.Log(LogLevel.Information, KnownLogEvents.None, _buffer[0]!, null, (state, exception) => state); _buffer.MoveNext(); } } } /// /// Try to match an event from the current buffer /// /// The matched event private List? MatchEvent() { LogEventMatch? currentMatch = null; foreach (ILogEventMatcher matcher in Matchers) { LogEventMatch? match = matcher.Match(_buffer); if(match != null) { if (currentMatch == null || match.Priority > currentMatch.Priority) { currentMatch = match; } } } return currentMatch?.Events; } /// /// Writes an event to the log /// /// The event to write protected virtual void WriteEvents(List logEvents) { foreach (LogEvent logEvent in logEvents) { _logger.Log(logEvent.Level, logEvent.Id, logEvent, null, (state, exception) => state.ToString()); } } } }