// Copyright Epic Games, Inc. All Rights Reserved.
//#define ENABLE_PUBLIC_DEBUG_CONTROLLER
#define ENABLE_SECURE_DEBUG_CONTROLLER
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using EpicGames.Core;
using Horde.Server.Acls;
using Horde.Server.Configuration;
using Horde.Server.Jobs;
using Horde.Server.Jobs.Graphs;
using Horde.Server.Jobs.Templates;
using Horde.Server.Logs;
using Horde.Server.Projects;
using Horde.Server.Utilities;
using JetBrains.Profiler.SelfApi;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
namespace Horde.Server.Server
{
#if ENABLE_PUBLIC_DEBUG_CONTROLLER
///
/// Public endpoints for the debug controller
///
[ApiController]
public class PublicDebugController : ControllerBase
{
///
/// The connection tracker service singleton
///
RequestTrackerService RequestTrackerService;
IHostApplicationLifetime ApplicationLifetime;
IDogStatsd DogStatsd;
///
/// Constructor
///
///
///
///
public PublicDebugController(RequestTrackerService RequestTrackerService, IHostApplicationLifetime ApplicationLifetime, IDogStatsd DogStatsd)
{
RequestTrackerService = RequestTrackerService;
ApplicationLifetime = ApplicationLifetime;
DogStatsd = DogStatsd;
}
///
/// Prints all the headers for the incoming request
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/headers")]
public IActionResult GetRequestHeaders()
{
StringBuilder Content = new StringBuilder();
Content.AppendLine("
");
foreach (KeyValuePair Pair in HttpContext.Request.Headers)
{
foreach (string Value in Pair.Value)
{
Content.AppendLine(HttpUtility.HtmlEncode($"{Pair.Key}: {Value}"));
}
}
Content.Append("
");
return new ContentResult { ContentType = "text/html", StatusCode = (int)HttpStatusCode.OK, Content = Content.ToString() };
}
///
/// Waits specified number of milliseconds and then returns a response
/// Used for testing timeouts proxy settings.
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/wait")]
public async Task GetAndWait([FromQuery] int WaitTimeMs = 1000)
{
await Task.Delay(WaitTimeMs);
string Content = $"Waited {WaitTimeMs} ms. " + new Random().Next(0, 10000000);
return new ContentResult { ContentType = "text/plain", StatusCode = (int)HttpStatusCode.OK, Content = Content };
}
///
/// Waits specified number of milliseconds and then throws an exception
/// Used for testing graceful shutdown and interruption of outstanding requests.
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/exception")]
public async Task ThrowException([FromQuery] int WaitTimeMs = 0)
{
await Task.Delay(WaitTimeMs);
throw new Exception("Test exception triggered by debug controller!");
}
///
/// Trigger an increment of a DogStatsd metric
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/metric")]
public ActionResult TriggerMetric([FromQuery] int Value = 10)
{
DogStatsd.Increment("hordeMetricTest", Value);
return Ok("Incremented metric 'hordeMetricTest' Type: " + DogStatsd.GetType());
}
///
/// Display metrics related to the .NET runtime
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/dotnet-metrics")]
public ActionResult DotNetMetrics()
{
ThreadPool.GetMaxThreads(out int MaxWorkerThreads, out int MaxIoThreads);
ThreadPool.GetAvailableThreads(out int FreeWorkerThreads, out int FreeIoThreads);
ThreadPool.GetMinThreads(out int MinWorkerThreads, out int MinIoThreads);
int BusyIoThreads = MaxIoThreads - FreeIoThreads;
int BusyWorkerThreads = MaxWorkerThreads - FreeWorkerThreads;
StringBuilder Content = new StringBuilder();
Content.AppendLine("Threads:");
Content.AppendLine("-------------------------------------------------------------");
Content.AppendLine("Worker busy={0,-5} free={1,-5} min={2,-5} max={3,-5}", BusyWorkerThreads, FreeWorkerThreads, MinWorkerThreads, MaxWorkerThreads);
Content.AppendLine(" IOCP busy={0,-5} free={1,-5} min={2,-5} max={3,-5}", BusyIoThreads, FreeIoThreads, MinIoThreads, MaxWorkerThreads);
NumberFormatInfo Nfi = (NumberFormatInfo)CultureInfo.InvariantCulture.NumberFormat.Clone();
Nfi.NumberGroupSeparator = " ";
string FormatBytes(long Number)
{
return (Number / 1024 / 1024).ToString("#,0", Nfi) + " MB";
}
GCMemoryInfo GcMemoryInfo = GC.GetGCMemoryInfo();
Content.AppendLine("");
Content.AppendLine("");
Content.AppendLine("Garbage collection (GC):");
Content.AppendLine("-------------------------------------------------------------");
Content.AppendLine(" Latency mode: " + GCSettings.LatencyMode);
Content.AppendLine(" Is server GC: " + GCSettings.IsServerGC);
Content.AppendLine(" Total memory: " + FormatBytes(GC.GetTotalMemory(false)));
Content.AppendLine(" Total allocated: " + FormatBytes(GC.GetTotalAllocatedBytes(false)));
Content.AppendLine(" Heap size: " + FormatBytes(GcMemoryInfo.HeapSizeBytes));
Content.AppendLine(" Fragmented: " + FormatBytes(GcMemoryInfo.FragmentedBytes));
Content.AppendLine(" Memory Load: " + FormatBytes(GcMemoryInfo.MemoryLoadBytes));
Content.AppendLine(" Total available memory: " + FormatBytes(GcMemoryInfo.TotalAvailableMemoryBytes));
Content.AppendLine("High memory load threshold: " + FormatBytes(GcMemoryInfo.HighMemoryLoadThresholdBytes));
return Ok(Content.ToString());
}
///
/// Force a full GC of all generations
///
/// Prints time taken in ms
[HttpGet]
[Route("/api/v1/debug/force-gc")]
public ActionResult ForceTriggerGc()
{
Stopwatch Timer = new Stopwatch();
Timer.Start();
GC.Collect();
Timer.Stop();
return Ok($"Time taken: {Timer.Elapsed.TotalMilliseconds} ms");
}
///
/// Lists requests in progress
///
/// HTML result
[HttpGet]
[Route("/api/v1/debug/requests-in-progress")]
public ActionResult GetRequestsInProgress()
{
StringBuilder Content = new StringBuilder();
Content.AppendLine("");
Content.AppendLine("
Requests in progress
");
Content.AppendLine("
");
Content.AppendLine("
");
Content.AppendLine("
Request Trace ID
");
Content.AppendLine("
Path
");
Content.AppendLine("
Started At
");
Content.AppendLine("
Age
");
Content.AppendLine("
");
List> Requests = RequestTrackerService.GetRequestsInProgress().ToList();
Requests.Sort((A, B) => A.Value.StartedAt.CompareTo(B.Value.StartedAt));
foreach (KeyValuePair Entry in Requests)
{
Content.Append("
");
Content.AppendLine($"
{Entry.Key}
");
Content.AppendLine($"
{Entry.Value.Request.Path}
");
Content.AppendLine($"
{Entry.Value.StartedAt}
");
Content.AppendLine($"
{Entry.Value.GetTimeSinceStartInMs()} ms
");
Content.Append("
");
}
Content.Append("
\n\n");
return new ContentResult { ContentType = "text/html", StatusCode = (int)HttpStatusCode.OK, Content = Content.ToString() };
}
/*
// Used during development only
[HttpGet]
[Route("/api/v1/debug/stop")]
public ActionResult StopApp()
{
Task.Run(async () =>
{
await Task.Delay(100);
ApplicationLifetime.StopApplication();
});
return new ContentResult { ContentType = "text/plain", StatusCode = (int)HttpStatusCode.OK, Content = "App stopping..." };
}
/**/
}
#endif
#if ENABLE_SECURE_DEBUG_CONTROLLER
///
/// Controller managing account status
///
[ApiController]
[Authorize]
public class SecureDebugController : HordeControllerBase
{
private static readonly Random s_random = new ();
private readonly MongoService _mongoService;
private readonly ConfigService _configService;
private readonly JobTaskSource _jobTaskSource;
private readonly IGraphCollection _graphCollection;
private readonly ILogFileCollection _logFileCollection;
private readonly IOptions _settings;
private readonly IOptionsSnapshot _globalConfig;
private readonly ILogger _logger;
///
/// Constructor
///
public SecureDebugController(MongoService mongoService, ConfigService configService, JobTaskSource jobTaskSource,
IGraphCollection graphCollection,
ILogFileCollection logFileCollection, IOptions settings, IOptionsSnapshot globalConfig, ILogger logger)
{
_mongoService = mongoService;
_configService = configService;
_jobTaskSource = jobTaskSource;
_graphCollection = graphCollection;
_logFileCollection = logFileCollection;
_settings = settings;
_globalConfig = globalConfig;
_logger = logger;
}
///
/// Prints all the environment variables
///
/// Http result
[HttpGet]
[Route("/api/v1/debug/environment")]
public ActionResult GetServerEnvVars()
{
if (!_globalConfig.Value.Authorize(ServerAclAction.Debug, User))
{
return Forbid(ServerAclAction.Debug);
}
StringBuilder content = new StringBuilder();
content.AppendLine("
");
foreach (System.Collections.DictionaryEntry? pair in System.Environment.GetEnvironmentVariables())
{
if (pair != null)
{
content.AppendLine(HttpUtility.HtmlEncode($"{pair.Value.Key}={pair.Value.Value}"));
}
}
content.Append("
");
return new ContentResult { ContentType = "text/html", StatusCode = (int)HttpStatusCode.OK, Content = content.ToString() };
}
///
/// Returns diagnostic information about the current state of the queue
///
/// Information about the queue
[HttpGet]
[Route("/api/v1/debug/queue")]
public ActionResult