// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Buffers; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.IO; using System.Linq; using System.Net.Http; using System.Security.Claims; using System.Text; using System.Text.Json; using System.Threading.Tasks; using HordeServer.Api; using HordeServer.Collections; using HordeServer.Logs; using HordeServer.Models; using HordeServer.Services; using HordeServer.Utilities; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using MongoDB.Bson; namespace HordeServer.Controllers { /// /// Format for the returned data /// public enum LogOutputFormat { /// /// Plain text /// Text, /// /// Raw output (text/json) /// Raw, } /// /// Controller for the /api/logs endpoint /// [ApiController] [Authorize] [Route("[controller]")] public class LogsController : ControllerBase { /// /// Instance of the LogFile service /// private readonly ILogFileService LogFileService; /// /// Instance of the issue collection /// private readonly IIssueCollection IssueCollection; /// /// Instance of the ACL service /// private readonly AclService AclService; /// /// Instance of the Job service /// private readonly JobService JobService; /// /// Constructor /// /// The Logfile service /// The issue collection /// The ACL service /// The Job service public LogsController(ILogFileService LogFileService, IIssueCollection IssueCollection, AclService AclService, JobService JobService) { this.LogFileService = LogFileService; this.IssueCollection = IssueCollection; this.AclService = AclService; this.JobService = JobService; } /// /// Creates a new logfile /// /// Parameters for the new LogFile /// Http result code [HttpPost] [Route("/api/v1/logs")] public async Task> CreateLogFile([FromBody] CreateLogFileRequest Create) { ObjectId JobId = Create.JobId.ToObjectId(); if (!await JobService.AuthorizeAsync(JobId, AclAction.CreateLog, User, null)) { return Forbid(); } ILogFile NewLogFile = await LogFileService.CreateLogFileAsync(JobId, null, Create.Type); return new CreateLogFileResponse(NewLogFile.Id.ToString()); } /// /// Retrieve metadata about a specific log file /// /// Id of the log file to get information about /// Filter for the properties to return /// Information about the requested project [HttpGet] [Route("/api/v1/logs/{LogFileId}")] [ProducesResponseType(typeof(GetLogFileResponse), 200)] public async Task> GetLog(string LogFileId, [FromQuery] PropertyFilter? Filter = null) { ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId()); if (LogFile == null) { return NotFound(); } if (!await AuthorizeAsync(LogFile, AclAction.ViewLog, User, null)) { return Forbid(); } LogMetadata Metadata = await LogFileService.GetMetadataAsync(LogFile); return new GetLogFileResponse(LogFile, Metadata).ApplyFilter(Filter); } /// /// Retrieve raw data for a log file /// /// Id of the log file to get information about /// Format for the returned data /// The log offset in bytes /// Number of bytes to return /// Name of the default filename to download /// Whether to download the file rather than display in the browser /// Raw log data for the requested range [HttpGet] [Route("/api/v1/logs/{LogFileId}/data")] public async Task GetLogData(string LogFileId, [FromQuery] LogOutputFormat Format = LogOutputFormat.Raw, [FromQuery] long Offset = 0, [FromQuery] long Length = long.MaxValue, [FromQuery] string? FileName = null, [FromQuery] bool Download = false) { ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId()); if (LogFile == null) { return NotFound(); } if (!await AuthorizeAsync(LogFile, AclAction.ViewLog, User, null)) { return Forbid(); } Func CopyTask; if (Format == LogOutputFormat.Text && LogFile.Type == LogType.Json) { CopyTask = (OutputStream, Context) => LogFileService.CopyPlainTextStreamAsync(LogFile, Offset, Length, OutputStream); } else { CopyTask = (OutputStream, Context) => LogFileService.CopyRawStreamAsync(LogFile, Offset, Length, OutputStream); } return new CustomFileCallbackResult(FileName ?? $"log-{LogFileId}.txt", "text/plain", !Download, CopyTask); } /// /// Retrieve line data for a logfile /// /// Id of the log file to get information about /// Index of the first line to retrieve /// Number of lines to retrieve /// Information about the requested project [HttpGet] [Route("/api/v1/logs/{LogFileId}/lines")] public async Task GetLogLines(string LogFileId, [FromQuery] int Index = 0, [FromQuery] int Count = int.MaxValue) { ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId()); if (LogFile == null) { return NotFound(); } if (!await AuthorizeAsync(LogFile, AclAction.ViewLog, User, null)) { return Forbid(); } LogMetadata Metadata = await LogFileService.GetMetadataAsync(LogFile); (int MinIndex, long MinOffset) = await LogFileService.GetLineOffsetAsync(LogFile, Index); (int MaxIndex, long MaxOffset) = await LogFileService.GetLineOffsetAsync(LogFile, Index + Math.Min(Count, int.MaxValue - Index)); Index = MinIndex; Count = MaxIndex - MinIndex; byte[] Result; using (System.IO.Stream Stream = await LogFileService.OpenRawStreamAsync(LogFile, MinOffset, MaxOffset - MinOffset)) { Result = new byte[Stream.Length]; await Stream.ReadFixedSizeDataAsync(Result, 0, Result.Length); } using (MemoryStream Stream = new MemoryStream(Result.Length + (Count * 20))) { Stream.WriteByte((byte)'{'); Stream.Write(Encoding.UTF8.GetBytes($"\"index\":{Index},")); Stream.Write(Encoding.UTF8.GetBytes($"\"count\":{Count},")); Stream.Write(Encoding.UTF8.GetBytes($"\"maxLineIndex\":{Metadata.MaxLineIndex},")); Stream.Write(Encoding.UTF8.GetBytes($"\"format\":{ (LogFile.Type == LogType.Json ? "\"JSON\"" : "\"TEXT\"")},")); // Stream.Write(Encoding.UTF8.GetBytes($"\"minIndex\":{MinIndex},")); // Stream.Write(Encoding.UTF8.GetBytes($"\"minOffset\":{MinOffset},")); // Stream.Write(Encoding.UTF8.GetBytes($"\"maxIndex\":{MaxIndex},")); // Stream.Write(Encoding.UTF8.GetBytes($"\"maxOffset\":{MaxOffset},")); // Stream.Write(Encoding.UTF8.GetBytes($"\"length\":{Result.Length},")); Stream.Write(Encoding.UTF8.GetBytes($"\"lines\":[")); Stream.WriteByte((byte)'\n'); int Offset = 0; for (int Line = Index; Line < Index + Count; Line++) { Stream.WriteByte((byte)' '); Stream.WriteByte((byte)' '); if (LogFile.Type == LogType.Json) { // Find the end of the line and output it as an opaque blob int StartOffset = Offset; for (; ; Offset++) { if (Offset == Result.Length) { Stream.WriteByte((byte)'{'); Stream.WriteByte((byte)'}'); break; } else if (Result[Offset] == (byte)'\n') { Stream.Write(Result, StartOffset, Offset - StartOffset); Offset++; break; } } } else { Stream.WriteByte((byte)'\"'); for (; Offset < Result.Length; Offset++) { if (Result[Offset] == '\\' || Result[Offset] == '\"') { Stream.WriteByte((byte)'\\'); Stream.WriteByte(Result[Offset]); } else if (Result[Offset] == (byte)'\n') { Offset++; break; } else if (Result[Offset] >= 32 && Result[Offset] <= 126) { Stream.WriteByte(Result[Offset]); } else { Stream.Write(Encoding.UTF8.GetBytes($"\\x{Result[Offset]:x2}")); } } Stream.WriteByte((byte)'\"'); } if (Line + 1 < Index + Count) { Stream.WriteByte((byte)','); } Stream.WriteByte((byte)'\n'); } if (LogFile.Type == LogType.Json) { Stream.Write(Encoding.UTF8.GetBytes($"]")); } Stream.WriteByte((byte)'}'); Response.ContentType = "application/json"; Response.Headers.ContentLength = Stream.Length; Stream.Position = 0; await Stream.CopyToAsync(Response.Body); } return new EmptyResult(); } /// /// Search log data /// /// Id of the log file to get information about /// Text to search for /// First line to search from /// Number of results to return /// Raw log data for the requested range [HttpGet] [Route("/api/v1/logs/{LogFileId}/search")] public async Task> SearchLogFileAsync(string LogFileId, [FromQuery] string Text, [FromQuery] int FirstLine = 0, [FromQuery] int Count = 5) { ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId()); if (LogFile == null) { return NotFound(); } if (!await AuthorizeAsync(LogFile, AclAction.ViewLog, User, null)) { return Forbid(); } SearchLogFileResponse Response = new SearchLogFileResponse(); Response.Stats = new LogSearchStats(); Response.Lines = await LogFileService.SearchLogDataAsync(LogFile, Text, FirstLine, Count, Response.Stats); return Response; } /// /// Retrieve events for a logfile /// /// Id of the log file to get information about /// Index of the first line to retrieve /// Number of lines to retrieve /// Information about the requested project [HttpGet] [Route("/api/v1/logs/{LogFileId}/events")] [ProducesResponseType(typeof(List), 200)] public async Task>> GetEventsAsync(string LogFileId, [FromQuery] int? Index = null, [FromQuery] int? Count = null) { ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId()); if (LogFile == null) { return NotFound(); } if (!await AuthorizeAsync(LogFile, AclAction.ViewLog, User, null)) { return Forbid(); } List LogEvents = await LogFileService.FindLogEventsAsync(LogFile, Index, Count); Dictionary SpanIdToIssueId = new Dictionary(); List Responses = new List(); foreach (ILogEvent LogEvent in LogEvents) { ILogEventData LogEventData = await LogFileService.GetEventDataAsync(LogFile, LogEvent.LineIndex, LogEvent.LineCount); int? IssueId = null; if (LogEvent.SpanId != null && !SpanIdToIssueId.TryGetValue(LogEvent.SpanId.Value, out IssueId)) { IIssueSpan? Span = await IssueCollection.GetSpanAsync(LogEvent.SpanId.Value); IssueId = Span?.IssueId; SpanIdToIssueId[LogEvent.SpanId.Value] = IssueId; } Responses.Add(new GetLogEventResponse(LogEvent, LogEventData, IssueId)); } return Responses; } /// /// Appends data to a log file /// /// The logfile id /// Offset within the log file /// The line index /// Http result code [HttpPost] [Route("/api/v1/logs/{LogFileId}")] public async Task WriteData(string LogFileId, [FromQuery] long Offset, [FromQuery] int LineIndex) { ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId()); if (LogFile == null) { return NotFound(); } if (!await AuthorizeAsync(LogFile, AclAction.WriteLogData, User, null)) { return Forbid(); } using (MemoryStream BodyStream = new MemoryStream()) { await Request.Body.CopyToAsync(BodyStream); await LogFileService.WriteLogDataAsync(LogFile, Offset, LineIndex, BodyStream.ToArray(), false); } return Ok(); } /// /// Determines if the user is authorized to perform an action on a particular template /// /// The template to check /// The action being performed /// The principal to authorize /// Permissions cache /// True if the action is authorized async Task AuthorizeAsync(ILogFile LogFile, AclAction Action, ClaimsPrincipal User, JobPermissionsCache? PermissionsCache) { if (LogFile.JobId != ObjectId.Empty && await JobService.AuthorizeAsync(LogFile.JobId, Action, User, PermissionsCache)) { return true; } if (LogFile.SessionId != null && await AclService.AuthorizeAsync(AclAction.ViewSession, User, PermissionsCache)) { return true; } return false; } } }