// Copyright Epic Games, Inc. All Rights Reserved.
//#define ENABLE_PUBLIC_DEBUG_CONTROLLER
#define ENABLE_SECURE_DEBUG_CONTROLLER
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using EpicGames.Core;
using Horde.Build.Acls;
using Horde.Build.Jobs;
using Horde.Build.Jobs.Graphs;
using Horde.Build.Jobs.Templates;
using Horde.Build.Logs;
using Horde.Build.Utilities;
using JetBrains.Profiler.SelfApi;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
namespace Horde.Build.Server
{
#if ENABLE_PUBLIC_DEBUG_CONTROLLER
///
/// Public endpoints for the debug controller
///
[ApiController]
public class PublicDebugController : ControllerBase
{
///
/// The connection tracker service singleton
///
RequestTrackerService RequestTrackerService;
IHostApplicationLifetime ApplicationLifetime;
IDogStatsd DogStatsd;
///
/// Constructor
///
///
///
///
public PublicDebugController(RequestTrackerService RequestTrackerService, IHostApplicationLifetime ApplicationLifetime, IDogStatsd DogStatsd)
{
RequestTrackerService = RequestTrackerService;
ApplicationLifetime = ApplicationLifetime;
DogStatsd = DogStatsd;
}
///
/// Prints all the headers for the incoming request
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/headers")]
public IActionResult GetRequestHeaders()
{
StringBuilder Content = new StringBuilder();
Content.AppendLine("");
foreach (KeyValuePair Pair in HttpContext.Request.Headers)
{
foreach (string Value in Pair.Value)
{
Content.AppendLine(HttpUtility.HtmlEncode($"{Pair.Key}: {Value}"));
}
}
Content.Append("
");
return new ContentResult { ContentType = "text/html", StatusCode = (int)HttpStatusCode.OK, Content = Content.ToString() };
}
///
/// Waits specified number of milliseconds and then returns a response
/// Used for testing timeouts proxy settings.
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/wait")]
public async Task GetAndWait([FromQuery] int WaitTimeMs = 1000)
{
await Task.Delay(WaitTimeMs);
string Content = $"Waited {WaitTimeMs} ms. " + new Random().Next(0, 10000000);
return new ContentResult { ContentType = "text/plain", StatusCode = (int)HttpStatusCode.OK, Content = Content };
}
///
/// Waits specified number of milliseconds and then throws an exception
/// Used for testing graceful shutdown and interruption of outstanding requests.
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/exception")]
public async Task ThrowException([FromQuery] int WaitTimeMs = 0)
{
await Task.Delay(WaitTimeMs);
throw new Exception("Test exception triggered by debug controller!");
}
///
/// Trigger an increment of a DogStatsd metric
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/metric")]
public ActionResult TriggerMetric([FromQuery] int Value = 10)
{
DogStatsd.Increment("hordeMetricTest", Value);
return Ok("Incremented metric 'hordeMetricTest' Type: " + DogStatsd.GetType());
}
///
/// Display metrics related to the .NET runtime
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/dotnet-metrics")]
public ActionResult DotNetMetrics()
{
ThreadPool.GetMaxThreads(out int MaxWorkerThreads, out int MaxIoThreads);
ThreadPool.GetAvailableThreads(out int FreeWorkerThreads, out int FreeIoThreads);
ThreadPool.GetMinThreads(out int MinWorkerThreads, out int MinIoThreads);
int BusyIoThreads = MaxIoThreads - FreeIoThreads;
int BusyWorkerThreads = MaxWorkerThreads - FreeWorkerThreads;
StringBuilder Content = new StringBuilder();
Content.AppendLine("Threads:");
Content.AppendLine("-------------------------------------------------------------");
Content.AppendLine("Worker busy={0,-5} free={1,-5} min={2,-5} max={3,-5}", BusyWorkerThreads, FreeWorkerThreads, MinWorkerThreads, MaxWorkerThreads);
Content.AppendLine(" IOCP busy={0,-5} free={1,-5} min={2,-5} max={3,-5}", BusyIoThreads, FreeIoThreads, MinIoThreads, MaxWorkerThreads);
NumberFormatInfo Nfi = (NumberFormatInfo)CultureInfo.InvariantCulture.NumberFormat.Clone();
Nfi.NumberGroupSeparator = " ";
string FormatBytes(long Number)
{
return (Number / 1024 / 1024).ToString("#,0", Nfi) + " MB";
}
GCMemoryInfo GcMemoryInfo = GC.GetGCMemoryInfo();
Content.AppendLine("");
Content.AppendLine("");
Content.AppendLine("Garbage collection (GC):");
Content.AppendLine("-------------------------------------------------------------");
Content.AppendLine(" Latency mode: " + GCSettings.LatencyMode);
Content.AppendLine(" Is server GC: " + GCSettings.IsServerGC);
Content.AppendLine(" Total memory: " + FormatBytes(GC.GetTotalMemory(false)));
Content.AppendLine(" Total allocated: " + FormatBytes(GC.GetTotalAllocatedBytes(false)));
Content.AppendLine(" Heap size: " + FormatBytes(GcMemoryInfo.HeapSizeBytes));
Content.AppendLine(" Fragmented: " + FormatBytes(GcMemoryInfo.FragmentedBytes));
Content.AppendLine(" Memory Load: " + FormatBytes(GcMemoryInfo.MemoryLoadBytes));
Content.AppendLine(" Total available memory: " + FormatBytes(GcMemoryInfo.TotalAvailableMemoryBytes));
Content.AppendLine("High memory load threshold: " + FormatBytes(GcMemoryInfo.HighMemoryLoadThresholdBytes));
return Ok(Content.ToString());
}
///
/// Force a full GC of all generations
///
/// Prints time taken in ms
[HttpGet]
[Route("/api/v1/debug/force-gc")]
public ActionResult ForceTriggerGc()
{
Stopwatch Timer = new Stopwatch();
Timer.Start();
GC.Collect();
Timer.Stop();
return Ok($"Time taken: {Timer.Elapsed.TotalMilliseconds} ms");
}
///
/// Lists requests in progress
///
/// HTML result
[HttpGet]
[Route("/api/v1/debug/requests-in-progress")]
public ActionResult GetRequestsInProgress()
{
StringBuilder Content = new StringBuilder();
Content.AppendLine("");
Content.AppendLine("Requests in progress
");
Content.AppendLine("");
Content.AppendLine("");
Content.AppendLine("| Request Trace ID | ");
Content.AppendLine("Path | ");
Content.AppendLine("Started At | ");
Content.AppendLine("Age | ");
Content.AppendLine("
");
List> Requests = RequestTrackerService.GetRequestsInProgress().ToList();
Requests.Sort((A, B) => A.Value.StartedAt.CompareTo(B.Value.StartedAt));
foreach (KeyValuePair Entry in Requests)
{
Content.Append("");
Content.AppendLine($"| {Entry.Key} | ");
Content.AppendLine($"{Entry.Value.Request.Path} | ");
Content.AppendLine($"{Entry.Value.StartedAt} | ");
Content.AppendLine($"{Entry.Value.GetTimeSinceStartInMs()} ms | ");
Content.Append("
");
}
Content.Append("
\n\n");
return new ContentResult { ContentType = "text/html", StatusCode = (int)HttpStatusCode.OK, Content = Content.ToString() };
}
/*
// Used during development only
[HttpGet]
[Route("/api/v1/debug/stop")]
public ActionResult StopApp()
{
Task.Run(async () =>
{
await Task.Delay(100);
ApplicationLifetime.StopApplication();
});
return new ContentResult { ContentType = "text/plain", StatusCode = (int)HttpStatusCode.OK, Content = "App stopping..." };
}
/**/
}
#endif
#if ENABLE_SECURE_DEBUG_CONTROLLER
///
/// Controller managing account status
///
[ApiController]
[Authorize]
public class SecureDebugController : ControllerBase
{
private static readonly Random s_random = new ();
///
/// The ACL service singleton
///
private readonly AclService _aclService;
///
/// The database service instance
///
private readonly MongoService _mongoService;
///
/// The job task source singleton
///
private readonly JobTaskSource _jobTaskSource;
///
/// Collection of template documents
///
private readonly ITemplateCollection _templateCollection;
///
/// The graph collection singleton
///
private readonly IGraphCollection _graphCollection;
///
/// The log file collection singleton
///
private readonly ILogFileCollection _logFileCollection;
///
/// Settings
///
private readonly IOptionsMonitor _settings;
///
/// Logger
///
private readonly ILogger _logger;
///
/// Constructor
///
/// The ACL service singleton
/// The database service instance
/// The dispatch service singleton
/// Collection of template documents
/// The graph collection
/// The log file collection
/// Settings
/// Logger
public SecureDebugController(AclService aclService, MongoService mongoService, JobTaskSource jobTaskSource,
ITemplateCollection templateCollection, IGraphCollection graphCollection,
ILogFileCollection logFileCollection, IOptionsMonitor settings, ILogger logger)
{
_aclService = aclService;
_mongoService = mongoService;
_jobTaskSource = jobTaskSource;
_templateCollection = templateCollection;
_graphCollection = graphCollection;
_logFileCollection = logFileCollection;
_settings = settings;
_logger = logger;
}
///
/// Prints all the environment variables
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/environment")]
public async Task GetServerEnvVars()
{
if (!await _aclService.AuthorizeAsync(AclAction.AdminRead, User))
{
return Forbid();
}
StringBuilder content = new StringBuilder();
content.AppendLine("");
foreach (System.Collections.DictionaryEntry? pair in System.Environment.GetEnvironmentVariables())
{
if (pair != null)
{
content.AppendLine(HttpUtility.HtmlEncode($"{pair.Value.Key}={pair.Value.Value}"));
}
}
content.Append("");
return new ContentResult { ContentType = "text/html", StatusCode = (int)HttpStatusCode.OK, Content = content.ToString() };
}
///
/// Returns diagnostic information about the current state of the queue
///
/// Information about the queue
[HttpGet]
[Route("/api/v1/debug/queue")]
public async Task> GetQueueStatusAsync()
{
if (!await _aclService.AuthorizeAsync(AclAction.AdminRead, User))
{
return Forbid();
}
return _jobTaskSource.GetStatus();
}
///
/// Returns the complete config Horde uses
///
/// Information about the config
[HttpGet]
[Route("/api/v1/debug/config")]
public async Task GetConfig()
{
if (!await _aclService.AuthorizeAsync(AclAction.AdminRead, User))
{
return Forbid();
}
JsonSerializerOptions options = new JsonSerializerOptions
{
WriteIndented = true
};
return Ok(JsonSerializer.Serialize(_settings.CurrentValue, options));
}
///
/// Generate log message of varying size
///
/// Information about the log message generated
[HttpGet]
[Route("/api/v1/debug/generate-log-msg")]
public async Task GenerateLogMessage(
[FromQuery] string? logLevel = null,
[FromQuery] int messageLen = 0,
[FromQuery] int exceptionMessageLen = 0,
[FromQuery] int argCount = 0,
[FromQuery] int argLen = 10)
{
if (!await _aclService.AuthorizeAsync(AclAction.AdminRead, User))
{
return Forbid();
}
string RandomString(int length)
{
const string Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
return new string(Enumerable.Repeat(Chars, length).Select(s => s[s_random.Next(s.Length)]).ToArray());
}
if (!Enum.TryParse(logLevel, out LogLevel logLevelInternal))
{
logLevelInternal = LogLevel.Information;
}
Exception? exception = null;
string message = "Message generated by /api/v1/debug/generate-log-msg";
message += RandomString(messageLen);
if (exceptionMessageLen > 0)
{
exception = new Exception("Exception from /api/v1/debug/generate-log-msg " + RandomString(exceptionMessageLen));
}
Dictionary args = new ();
if (argCount > 0)
{
for (int i = 0; i < argCount; i++)
{
args["Arg" + i] = "Arg 1 - " + RandomString(argLen);
}
}
using IDisposable logScope = _logger.BeginScope(args);
// Ignore warning as we explicitly want to build this message manually
#pragma warning disable CA2254 // Template should be a static expression
_logger.Log(logLevelInternal, exception, message);
#pragma warning restore CA2254
return Ok($"Log message generated logLevel={logLevelInternal} messageLen={messageLen} exceptionMessageLen={exceptionMessageLen} argCount={argCount} argLen={argLen}");
}
///
/// Queries for all graphs
///
/// The graph definitions
[HttpGet]
[Route("/api/v1/debug/graphs")]
[ProducesResponseType(200, Type = typeof(GetGraphResponse))]
public async Task>> GetGraphsAsync([FromQuery] int? index = null, [FromQuery] int? count = null, [FromQuery] PropertyFilter? filter = null)
{
if (!await _aclService.AuthorizeAsync(AclAction.AdminRead, User))
{
return Forbid();
}
List graphs = await _graphCollection.FindAllAsync(null, index, count);
return graphs.ConvertAll(x => new GetGraphResponse(x).ApplyFilter(filter));
}
///
/// Queries for a particular graph by hash
///
/// The graph definition
[HttpGet]
[Route("/api/v1/debug/graphs/{GraphId}")]
[ProducesResponseType(200, Type = typeof(GetGraphResponse))]
public async Task> GetGraphAsync(string graphId, [FromQuery] PropertyFilter? filter = null)
{
if (!await _aclService.AuthorizeAsync(AclAction.AdminRead, User))
{
return Forbid();
}
IGraph graph = await _graphCollection.GetAsync(ContentHash.Parse(graphId));
return new GetGraphResponse(graph).ApplyFilter(filter);
}
///
/// Query all the job templates.
///
/// Filter for properties to return
/// Information about all the job templates
[HttpGet]
[Route("/api/v1/debug/templates")]
[ProducesResponseType(typeof(List), 200)]
public async Task> GetTemplatesAsync([FromQuery] PropertyFilter? filter = null)
{
if (!await _aclService.AuthorizeAsync(AclAction.AdminRead, User))
{
return Forbid();
}
List templates = await _templateCollection.FindAllAsync();
return templates.ConvertAll(x => new GetTemplateResponse(x).ApplyFilter(filter));
}
///
/// Retrieve information about a specific job template.
///
/// Id of the template to get information about
/// List of properties to return
/// Information about the requested template
[HttpGet]
[Route("/api/v1/debug/templates/{TemplateHash}")]
[ProducesResponseType(typeof(GetTemplateResponse), 200)]
public async Task> GetTemplateAsync(string templateHash, [FromQuery] PropertyFilter? filter = null)
{
ContentHash templateHashValue = ContentHash.Parse(templateHash);
if (!await _aclService.AuthorizeAsync(AclAction.AdminRead, User))
{
return Forbid();
}
ITemplate? template = await _templateCollection.GetAsync(templateHashValue);
if (template == null)
{
return NotFound();
}
return template.ApplyFilter(filter);
}
///
/// Retrieve metadata about a specific log file
///
/// Id of the log file to get information about
/// Filter for the properties to return
/// Information about the requested project
[HttpGet]
[Route("/api/v1/debug/logs/{LogFileId}")]
public async Task> GetLogAsync(ObjectId logFileId, [FromQuery] PropertyFilter? filter = null)
{
if (!await _aclService.AuthorizeAsync(AclAction.AdminRead, User))
{
return Forbid();
}
ILogFile? logFile = await _logFileCollection.GetLogFileAsync(logFileId, CancellationToken.None);
if (logFile == null)
{
return NotFound();
}
return logFile.ApplyFilter(filter);
}
///
/// Populate the database with test data
///
/// Async task
[HttpGet]
[Route("/api/v1/debug/collections/{Name}")]
public async Task> GetDocumentsAsync(string name, [FromQuery] string? filter = null, [FromQuery] int index = 0, [FromQuery] int count = 10)
{
if (!await _aclService.AuthorizeAsync(AclAction.AdminRead, User))
{
return Forbid();
}
IMongoCollection> collection = _mongoService.GetCollection>(name);
List> documents = await collection.Find(filter ?? "{}").Skip(index).Limit(count).ToListAsync();
return documents;
}
///
/// Starts the profiler session
///
/// Text message
[HttpGet]
[Route("/api/v1/debug/profiler/start")]
public async Task StartProfiler()
{
await DotTrace.EnsurePrerequisiteAsync();
string snapshotDir = Path.Join(Path.GetTempPath(), "horde-profiler-snapshots");
if (!Directory.Exists(snapshotDir))
{
Directory.CreateDirectory(snapshotDir);
}
DotTrace.Config config = new ();
config.SaveToDir(snapshotDir);
DotTrace.Attach(config);
DotTrace.StartCollectingData();
return new ContentResult { ContentType = "text/plain", StatusCode = (int)HttpStatusCode.OK, Content = "Profiling session started. Using dir " + snapshotDir };
}
///
/// Stops the profiler session
///
/// Text message
[HttpGet]
[Route("/api/v1/debug/profiler/stop")]
public ActionResult StopProfiler()
{
DotTrace.SaveData();
DotTrace.Detach();
return new ContentResult { ContentType = "text/plain", StatusCode = (int)HttpStatusCode.OK, Content = "Profiling session stopped" };
}
///
/// Downloads the captured profiling snapshots
///
/// A .zip file containing the profiling snapshots
[HttpGet]
[Route("/api/v1/debug/profiler/download")]
public ActionResult DownloadProfilingData()
{
string snapshotZipFile = DotTrace.GetCollectedSnapshotFilesArchive(false);
if (!System.IO.File.Exists(snapshotZipFile))
{
return NotFound("The generated snapshot .zip file was not found");
}
return PhysicalFile(snapshotZipFile, "application/zip", Path.GetFileName(snapshotZipFile));
}
}
}
#endif