// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Buffers;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Security.Claims;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using HordeServer.Api;
using HordeServer.Collections;
using HordeServer.Logs;
using HordeServer.Models;
using HordeServer.Services;
using HordeServer.Utilities;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using MongoDB.Bson;
namespace HordeServer.Controllers
{
///
/// Format for the returned data
///
public enum LogOutputFormat
{
///
/// Plain text
///
Text,
///
/// Raw output (text/json)
///
Raw,
}
///
/// Controller for the /api/logs endpoint
///
[ApiController]
[Authorize]
[Route("[controller]")]
public class LogsController : ControllerBase
{
///
/// Instance of the LogFile service
///
private readonly ILogFileService LogFileService;
///
/// Instance of the issue collection
///
private readonly IIssueCollection IssueCollection;
///
/// Instance of the ACL service
///
private readonly AclService AclService;
///
/// Instance of the Job service
///
private readonly JobService JobService;
///
/// Constructor
///
/// The Logfile service
/// The issue collection
/// The ACL service
/// The Job service
public LogsController(ILogFileService LogFileService, IIssueCollection IssueCollection, AclService AclService, JobService JobService)
{
this.LogFileService = LogFileService;
this.IssueCollection = IssueCollection;
this.AclService = AclService;
this.JobService = JobService;
}
///
/// Creates a new logfile
///
/// Parameters for the new LogFile
/// Http result code
[HttpPost]
[Route("/api/v1/logs")]
public async Task> CreateLogFile([FromBody] CreateLogFileRequest Create)
{
ObjectId JobId = Create.JobId.ToObjectId();
if (!await JobService.AuthorizeAsync(JobId, AclAction.CreateLog, User, null))
{
return Forbid();
}
ILogFile NewLogFile = await LogFileService.CreateLogFileAsync(JobId, null, Create.Type);
return new CreateLogFileResponse(NewLogFile.Id.ToString());
}
///
/// Retrieve metadata about a specific log file
///
/// Id of the log file to get information about
/// Filter for the properties to return
/// Information about the requested project
[HttpGet]
[Route("/api/v1/logs/{LogFileId}")]
[ProducesResponseType(typeof(GetLogFileResponse), 200)]
public async Task> GetLog(string LogFileId, [FromQuery] PropertyFilter? Filter = null)
{
ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId());
if (LogFile == null)
{
return NotFound();
}
if (!await AuthorizeAsync(LogFile, AclAction.ViewLog, User, null))
{
return Forbid();
}
LogMetadata Metadata = await LogFileService.GetMetadataAsync(LogFile);
return new GetLogFileResponse(LogFile, Metadata).ApplyFilter(Filter);
}
///
/// Retrieve raw data for a log file
///
/// Id of the log file to get information about
/// Format for the returned data
/// The log offset in bytes
/// Number of bytes to return
/// Name of the default filename to download
/// Whether to download the file rather than display in the browser
/// Raw log data for the requested range
[HttpGet]
[Route("/api/v1/logs/{LogFileId}/data")]
public async Task GetLogData(string LogFileId, [FromQuery] LogOutputFormat Format = LogOutputFormat.Raw, [FromQuery] long Offset = 0, [FromQuery] long Length = long.MaxValue, [FromQuery] string? FileName = null, [FromQuery] bool Download = false)
{
ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId());
if (LogFile == null)
{
return NotFound();
}
if (!await AuthorizeAsync(LogFile, AclAction.ViewLog, User, null))
{
return Forbid();
}
Func CopyTask;
if (Format == LogOutputFormat.Text && LogFile.Type == LogType.Json)
{
CopyTask = (OutputStream, Context) => LogFileService.CopyPlainTextStreamAsync(LogFile, Offset, Length, OutputStream);
}
else
{
CopyTask = (OutputStream, Context) => LogFileService.CopyRawStreamAsync(LogFile, Offset, Length, OutputStream);
}
return new CustomFileCallbackResult(FileName ?? $"log-{LogFileId}.txt", "text/plain", !Download, CopyTask);
}
///
/// Retrieve line data for a logfile
///
/// Id of the log file to get information about
/// Index of the first line to retrieve
/// Number of lines to retrieve
/// Information about the requested project
[HttpGet]
[Route("/api/v1/logs/{LogFileId}/lines")]
public async Task GetLogLines(string LogFileId, [FromQuery] int Index = 0, [FromQuery] int Count = int.MaxValue)
{
ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId());
if (LogFile == null)
{
return NotFound();
}
if (!await AuthorizeAsync(LogFile, AclAction.ViewLog, User, null))
{
return Forbid();
}
LogMetadata Metadata = await LogFileService.GetMetadataAsync(LogFile);
(int MinIndex, long MinOffset) = await LogFileService.GetLineOffsetAsync(LogFile, Index);
(int MaxIndex, long MaxOffset) = await LogFileService.GetLineOffsetAsync(LogFile, Index + Math.Min(Count, int.MaxValue - Index));
Index = MinIndex;
Count = MaxIndex - MinIndex;
byte[] Result;
using (System.IO.Stream Stream = await LogFileService.OpenRawStreamAsync(LogFile, MinOffset, MaxOffset - MinOffset))
{
Result = new byte[Stream.Length];
await Stream.ReadFixedSizeDataAsync(Result, 0, Result.Length);
}
using (MemoryStream Stream = new MemoryStream(Result.Length + (Count * 20)))
{
Stream.WriteByte((byte)'{');
Stream.Write(Encoding.UTF8.GetBytes($"\"index\":{Index},"));
Stream.Write(Encoding.UTF8.GetBytes($"\"count\":{Count},"));
Stream.Write(Encoding.UTF8.GetBytes($"\"maxLineIndex\":{Metadata.MaxLineIndex},"));
Stream.Write(Encoding.UTF8.GetBytes($"\"format\":{ (LogFile.Type == LogType.Json ? "\"JSON\"" : "\"TEXT\"")},"));
// Stream.Write(Encoding.UTF8.GetBytes($"\"minIndex\":{MinIndex},"));
// Stream.Write(Encoding.UTF8.GetBytes($"\"minOffset\":{MinOffset},"));
// Stream.Write(Encoding.UTF8.GetBytes($"\"maxIndex\":{MaxIndex},"));
// Stream.Write(Encoding.UTF8.GetBytes($"\"maxOffset\":{MaxOffset},"));
// Stream.Write(Encoding.UTF8.GetBytes($"\"length\":{Result.Length},"));
Stream.Write(Encoding.UTF8.GetBytes($"\"lines\":["));
Stream.WriteByte((byte)'\n');
int Offset = 0;
for (int Line = Index; Line < Index + Count; Line++)
{
Stream.WriteByte((byte)' ');
Stream.WriteByte((byte)' ');
if (LogFile.Type == LogType.Json)
{
// Find the end of the line and output it as an opaque blob
int StartOffset = Offset;
for (; ; Offset++)
{
if (Offset == Result.Length)
{
Stream.WriteByte((byte)'{');
Stream.WriteByte((byte)'}');
break;
}
else if (Result[Offset] == (byte)'\n')
{
Stream.Write(Result, StartOffset, Offset - StartOffset);
Offset++;
break;
}
}
}
else
{
Stream.WriteByte((byte)'\"');
for (; Offset < Result.Length; Offset++)
{
if (Result[Offset] == '\\' || Result[Offset] == '\"')
{
Stream.WriteByte((byte)'\\');
Stream.WriteByte(Result[Offset]);
}
else if (Result[Offset] == (byte)'\n')
{
Offset++;
break;
}
else if (Result[Offset] >= 32 && Result[Offset] <= 126)
{
Stream.WriteByte(Result[Offset]);
}
else
{
Stream.Write(Encoding.UTF8.GetBytes($"\\x{Result[Offset]:x2}"));
}
}
Stream.WriteByte((byte)'\"');
}
if (Line + 1 < Index + Count)
{
Stream.WriteByte((byte)',');
}
Stream.WriteByte((byte)'\n');
}
if (LogFile.Type == LogType.Json)
{
Stream.Write(Encoding.UTF8.GetBytes($"]"));
}
Stream.WriteByte((byte)'}');
Response.ContentType = "application/json";
Response.Headers.ContentLength = Stream.Length;
Stream.Position = 0;
await Stream.CopyToAsync(Response.Body);
}
return new EmptyResult();
}
///
/// Search log data
///
/// Id of the log file to get information about
/// Text to search for
/// First line to search from
/// Number of results to return
/// Raw log data for the requested range
[HttpGet]
[Route("/api/v1/logs/{LogFileId}/search")]
public async Task> SearchLogFileAsync(string LogFileId, [FromQuery] string Text, [FromQuery] int FirstLine = 0, [FromQuery] int Count = 5)
{
ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId());
if (LogFile == null)
{
return NotFound();
}
if (!await AuthorizeAsync(LogFile, AclAction.ViewLog, User, null))
{
return Forbid();
}
SearchLogFileResponse Response = new SearchLogFileResponse();
Response.Stats = new LogSearchStats();
Response.Lines = await LogFileService.SearchLogDataAsync(LogFile, Text, FirstLine, Count, Response.Stats);
return Response;
}
///
/// Retrieve events for a logfile
///
/// Id of the log file to get information about
/// Index of the first line to retrieve
/// Number of lines to retrieve
/// Information about the requested project
[HttpGet]
[Route("/api/v1/logs/{LogFileId}/events")]
[ProducesResponseType(typeof(List), 200)]
public async Task>> GetEventsAsync(string LogFileId, [FromQuery] int? Index = null, [FromQuery] int? Count = null)
{
ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId());
if (LogFile == null)
{
return NotFound();
}
if (!await AuthorizeAsync(LogFile, AclAction.ViewLog, User, null))
{
return Forbid();
}
List LogEvents = await LogFileService.FindLogEventsAsync(LogFile, Index, Count);
Dictionary SpanIdToIssueId = new Dictionary();
List Responses = new List();
foreach (ILogEvent LogEvent in LogEvents)
{
ILogEventData LogEventData = await LogFileService.GetEventDataAsync(LogFile, LogEvent.LineIndex, LogEvent.LineCount);
int? IssueId = null;
if (LogEvent.SpanId != null && !SpanIdToIssueId.TryGetValue(LogEvent.SpanId.Value, out IssueId))
{
IIssueSpan? Span = await IssueCollection.GetSpanAsync(LogEvent.SpanId.Value);
IssueId = Span?.IssueId;
SpanIdToIssueId[LogEvent.SpanId.Value] = IssueId;
}
Responses.Add(new GetLogEventResponse(LogEvent, LogEventData, IssueId));
}
return Responses;
}
///
/// Appends data to a log file
///
/// The logfile id
/// Offset within the log file
/// The line index
/// Http result code
[HttpPost]
[Route("/api/v1/logs/{LogFileId}")]
public async Task WriteData(string LogFileId, [FromQuery] long Offset, [FromQuery] int LineIndex)
{
ILogFile? LogFile = await LogFileService.GetLogFileAsync(LogFileId.ToObjectId());
if (LogFile == null)
{
return NotFound();
}
if (!await AuthorizeAsync(LogFile, AclAction.WriteLogData, User, null))
{
return Forbid();
}
using (MemoryStream BodyStream = new MemoryStream())
{
await Request.Body.CopyToAsync(BodyStream);
await LogFileService.WriteLogDataAsync(LogFile, Offset, LineIndex, BodyStream.ToArray(), false);
}
return Ok();
}
///
/// Determines if the user is authorized to perform an action on a particular template
///
/// The template to check
/// The action being performed
/// The principal to authorize
/// Permissions cache
/// True if the action is authorized
async Task AuthorizeAsync(ILogFile LogFile, AclAction Action, ClaimsPrincipal User, JobPermissionsCache? PermissionsCache)
{
if (LogFile.JobId != ObjectId.Empty && await JobService.AuthorizeAsync(LogFile.JobId, Action, User, PermissionsCache))
{
return true;
}
if (LogFile.SessionId != null && await AclService.AuthorizeAsync(AclAction.ViewSession, User, PermissionsCache))
{
return true;
}
return false;
}
}
}