// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
namespace EpicGames.Core
{
///
/// Confidence of a matched log event being the correct derivation
///
public enum LogEventPriority
{
None,
Lowest,
Low,
BelowNormal,
Normal,
AboveNormal,
High,
Highest,
}
public class LogEventMatch
{
public LogEventPriority Priority { get; }
public List Events { get; }
public LogEventMatch(LogEventPriority priority, LogEvent logEvent)
{
Priority = priority;
Events = new List { logEvent };
}
public LogEventMatch(LogEventPriority priority, IEnumerable events)
{
Priority = priority;
Events = events.ToList();
}
}
///
/// Interface for a class which matches error strings
///
public interface ILogEventMatcher
{
///
/// Attempt to match events from the given input buffer
///
/// The input buffer
/// Information about the error that was matched, or null if an error was not matched
LogEventMatch? Match(ILogCursor cursor);
}
///
/// Turns raw text output into structured logging events
///
public class LogEventParser : IDisposable
{
///
/// List of event matchers for this parser
///
public List Matchers { get; } = new List();
///
/// List of patterns to ignore
///
public List IgnorePatterns { get; } = new List();
///
/// Buffer of input lines
///
readonly LogBuffer _buffer;
///
/// Buffer for holding partial line data
///
readonly MemoryStream _partialLine = new MemoryStream();
///
/// Whether matching is currently enabled
///
int _matchingEnabled;
///
/// The inner logger
///
ILogger _logger;
///
/// Public accessor for the logger
///
public ILogger Logger
{
get => _logger;
set => _logger = value;
}
///
/// Constructor
///
/// The logger to receive parsed output messages
public LogEventParser(ILogger logger)
{
_logger = logger;
_buffer = new LogBuffer(50);
}
///
public void Dispose() => Flush();
///
/// Enumerate all the types that implement in the given assembly, and create instances of them
///
/// The assembly to enumerate matchers from
public void AddMatchersFromAssembly(Assembly assembly)
{
foreach (Type type in assembly.GetTypes())
{
if (type.IsClass && typeof(ILogEventMatcher).IsAssignableFrom(type))
{
_logger.LogDebug("Adding event matcher: {Type}", type.Name);
ILogEventMatcher matcher = (ILogEventMatcher)Activator.CreateInstance(type)!;
Matchers.Add(matcher);
}
}
}
///
/// Writes a line to the event filter
///
/// The line to output
public void WriteLine(string line)
{
if (line.Length > 0 && line[0] == '{')
{
byte[] data = Encoding.UTF8.GetBytes(line);
try
{
JsonLogEvent jsonEvent;
if (JsonLogEvent.TryParse(data, out jsonEvent))
{
ProcessData(true);
_logger.Log(jsonEvent.Level, jsonEvent.EventId, jsonEvent, null, JsonLogEvent.Format);
return;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Exception while parsing log event");
}
}
_buffer.AddLine(StringUtils.ParseEscapeCodes(line));
ProcessData(false);
}
///
/// Writes data to the log parser
///
/// Data to write
public void WriteData(ReadOnlyMemory data)
{
int baseIdx = 0;
int scanIdx = 0;
ReadOnlySpan span = data.Span;
// Handle a partially existing line
if (_partialLine.Length > 0)
{
for (; scanIdx < span.Length; scanIdx++)
{
if (span[scanIdx] == '\n')
{
_partialLine.Write(span.Slice(baseIdx, scanIdx - baseIdx));
FlushPartialLine();
baseIdx = ++scanIdx;
break;
}
}
}
// Handle any complete lines
for (; scanIdx < span.Length; scanIdx++)
{
if(span[scanIdx] == '\n')
{
AddLine(data.Slice(baseIdx, scanIdx - baseIdx));
baseIdx = scanIdx + 1;
}
}
// Add the rest of the text to the partial line buffer
_partialLine.Write(span.Slice(baseIdx));
// Process the new data
ProcessData(false);
}
///
/// Flushes the current contents of the parser
///
public void Flush()
{
// If there's a partially written line, write that out first
if (_partialLine.Length > 0)
{
FlushPartialLine();
}
// Process any remaining data
ProcessData(true);
}
///
/// Adds a raw utf-8 string to the buffer
///
/// The string data
private void AddLine(ReadOnlyMemory data)
{
if (data.Length > 0 && data.Span[data.Length - 1] == '\r')
{
data = data.Slice(0, data.Length - 1);
}
if (data.Length > 0 && data.Span[0] == '{')
{
JsonLogEvent jsonEvent;
if (JsonLogEvent.TryParse(data, out jsonEvent))
{
ProcessData(true);
_logger.Log(jsonEvent.Level, jsonEvent.EventId, jsonEvent, null, JsonLogEvent.Format);
return;
}
}
_buffer.AddLine(StringUtils.ParseEscapeCodes(Encoding.UTF8.GetString(data.Span)));
}
///
/// Writes the current partial line data, with the given data appended to it, then clear the buffer
///
private void FlushPartialLine()
{
AddLine(_partialLine.ToArray());
_partialLine.Position = 0;
_partialLine.SetLength(0);
}
///
/// Process any data in the buffer
///
/// Whether we've reached the end of the stream
void ProcessData(bool bFlush)
{
while (_buffer.Length > 0)
{
// Try to match an event
List? events = null;
if (Regex.IsMatch(_buffer[0], "<-- Suspend Log Parsing -->", RegexOptions.IgnoreCase))
{
_matchingEnabled--;
}
else if (Regex.IsMatch(_buffer[0], "<-- Resume Log Parsing -->", RegexOptions.IgnoreCase))
{
_matchingEnabled++;
}
else if (_matchingEnabled >= 0)
{
events = MatchEvent();
}
// Bail out if we need more data
if (_buffer.Length < 1024 && !bFlush && _buffer.NeedMoreData)
{
break;
}
// If we did match something, check if it's not negated by an ignore pattern. We typically have relatively few errors and many more ignore patterns than matchers, so it's quicker
// to check them in response to an identified error than to treat them as matchers of their own.
if (events != null)
{
foreach (Regex ignorePattern in IgnorePatterns)
{
if (ignorePattern.IsMatch(_buffer[0]))
{
events = null;
break;
}
}
}
// Report the error to the listeners
if (events != null)
{
WriteEvents(events);
_buffer.Advance(events.Count);
}
else
{
_logger.Log(LogLevel.Information, KnownLogEvents.None, _buffer[0]!, null, (state, exception) => state);
_buffer.MoveNext();
}
}
}
///
/// Try to match an event from the current buffer
///
/// The matched event
private List? MatchEvent()
{
LogEventMatch? currentMatch = null;
foreach (ILogEventMatcher matcher in Matchers)
{
LogEventMatch? match = matcher.Match(_buffer);
if(match != null)
{
if (currentMatch == null || match.Priority > currentMatch.Priority)
{
currentMatch = match;
}
}
}
return currentMatch?.Events;
}
///
/// Writes an event to the log
///
/// The event to write
protected virtual void WriteEvents(List logEvents)
{
foreach (LogEvent logEvent in logEvents)
{
_logger.Log(logEvent.Level, logEvent.Id, logEvent, null, (state, exception) => state.ToString());
}
}
}
}