izzy/UnrealEngineUWP
0
0
Fork 0
mirror of https://github.com/izzy2lost/UnrealEngineUWP.git synced 2026-03-26 18:15:20 -07:00
Code Issues Packages Projects Releases Wiki Activity

Horde: Move storage system into a plugin.

Browse Source 
#rnx

[CL 34683622 by ben marsh in ue5-main branch]
This commit is contained in:
ben marsh
2024-06-26 14:16:36 -04:00
parent fb3a9b88d7
commit d8d1528669
40 changed files with 432 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
Engine/Source/Programs/Horde/Horde.sln
View File

@@ -144,7 +144,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HordeServer.Analytics", "Pl
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HordeServer.Analytics.Tests", "Plugins\Analytics\HordeServer.Analytics.Tests\HordeServer.Analytics.Tests.csproj", "{65A93DE6-CA55-421E-9D95-37A6A7897A08}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HordeServer.Tests.Shared", "HordeServer.Tests.Shared\HordeServer.Tests.Shared.csproj", "{2C0C4317-7C0F-4D95-AE97-B1F15FFDE631}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HordeServer.Tests.Shared", "HordeServer.Tests.Shared\HordeServer.Tests.Shared.csproj", "{2C0C4317-7C0F-4D95-AE97-B1F15FFDE631}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Storage", "Storage", "{E0EF6915-EB92-4402-B384-F1E2668EDEFB}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HordeServer.Storage", "Plugins\Storage\HordeServer.Storage\HordeServer.Storage.csproj", "{47012482-480E-403C-B6CC-3A633653870E}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HordeServer.Storage.Tests", "Plugins\Storage\HordeServer.Storage.Tests\HordeServer.Storage.Tests.csproj", "{31F78551-BBEB-4FF4-B61D-767319042E8B}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -780,6 +786,30 @@ Global
{2C0C4317-7C0F-4D95-AE97-B1F15FFDE631}.Release|Any CPU.Build.0 = Release|Any CPU
{2C0C4317-7C0F-4D95-AE97-B1F15FFDE631}.ReleaseAutoUpdate|Any CPU.ActiveCfg = Release|Any CPU
{2C0C4317-7C0F-4D95-AE97-B1F15FFDE631}.ReleaseAutoUpdate|Any CPU.Build.0 = Release|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.Analyze|Any CPU.ActiveCfg = Debug|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.Analyze|Any CPU.Build.0 = Debug|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.DebugAutoUpdate|Any CPU.ActiveCfg = Debug|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.DebugAutoUpdate|Any CPU.Build.0 = Debug|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.Development|Any CPU.ActiveCfg = Debug|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.Development|Any CPU.Build.0 = Debug|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.Release|Any CPU.Build.0 = Release|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.ReleaseAutoUpdate|Any CPU.ActiveCfg = Release|Any CPU
{47012482-480E-403C-B6CC-3A633653870E}.ReleaseAutoUpdate|Any CPU.Build.0 = Release|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.Analyze|Any CPU.ActiveCfg = Debug|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.Analyze|Any CPU.Build.0 = Debug|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.DebugAutoUpdate|Any CPU.ActiveCfg = Debug|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.DebugAutoUpdate|Any CPU.Build.0 = Debug|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.Development|Any CPU.ActiveCfg = Debug|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.Development|Any CPU.Build.0 = Debug|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.Release|Any CPU.Build.0 = Release|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.ReleaseAutoUpdate|Any CPU.ActiveCfg = Release|Any CPU
{31F78551-BBEB-4FF4-B61D-767319042E8B}.ReleaseAutoUpdate|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -833,6 +863,9 @@ Global
{71B9D9DB-7CAD-4E01-8889-04939AD1B949} = {C0214E80-0A85-4432-834D-9DC48104AD26}
{CDB902A7-5CDC-4B17-9DB0-ECB606FE0C31} = {71B9D9DB-7CAD-4E01-8889-04939AD1B949}
{65A93DE6-CA55-421E-9D95-37A6A7897A08} = {71B9D9DB-7CAD-4E01-8889-04939AD1B949}
{E0EF6915-EB92-4402-B384-F1E2668EDEFB} = {C0214E80-0A85-4432-834D-9DC48104AD26}
{47012482-480E-403C-B6CC-3A633653870E} = {E0EF6915-EB92-4402-B384-F1E2668EDEFB}
{31F78551-BBEB-4FF4-B61D-767319042E8B} = {E0EF6915-EB92-4402-B384-F1E2668EDEFB}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {F06D2583-1509-4254-BEE6-5921340022BF}

11
Engine/Source/Programs/Horde/HordeServer.Shared/Configuration/ConfigNode.cs
View File

@@ -48,6 +48,11 @@ namespace HordeServer.Configuration
/// </summary>
public abstract class ConfigNode
{
/// <summary>
/// Default options for new json node objects
/// </summary>
internal static JsonNodeOptions s_defaultJsonNodeOptions { get; } = new JsonNodeOptions { PropertyNameCaseInsensitive = true };
/// <summary>
/// Parses macro definitions from this object
/// </summary>
@@ -120,7 +125,7 @@ namespace HordeServer.Configuration
}
else if (node is JsonObject obj)
{
JsonObject result = new JsonObject();
JsonObject result = new JsonObject(s_defaultJsonNodeOptions);
foreach ((string propertyName, JsonNode? propertyNode) in obj)
{
result[propertyName] = ExpandMacros(propertyNode, context);
@@ -321,7 +326,7 @@ namespace HordeServer.Configuration
/// <inheritdoc/>
public override async Task<JsonNode?> PreprocessAsync(JsonNode? node, JsonNode? existingNode, ConfigContext context, CancellationToken cancellationToken)
{
JsonObject? targetObject = ((JsonObject?)existingNode) ?? new JsonObject();
JsonObject? targetObject = ((JsonObject?)existingNode) ?? new JsonObject(s_defaultJsonNodeOptions);
foreach ((string key, JsonNode? element) in (JsonObject)node!)
{
context.EnterScope($"[{key}]");
@@ -545,7 +550,7 @@ namespace HordeServer.Configuration
}
// Ensure that the target object is valid so we can write properties into it
target ??= new JsonObject();
target ??= new JsonObject(s_defaultJsonNodeOptions);
// Parse all the macros for this scope
if (MacroScope)

2
Engine/Source/Programs/Horde/HordeServer.Shared/HordeServer.Shared.csproj
View File

@@ -20,6 +20,8 @@
<ItemGroup>
<PackageReference Include="MongoDB.Driver" Version="2.22.0" />
<PackageReference Include="OpenTelemetry" Version="1.8.0" />
<PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.8.0" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.8.0" />
</ItemGroup>
</Project>

5
Engine/Source/Programs/Horde/HordeServer.Tests.Shared/ServerServiceTest.cs
View File

@@ -20,6 +20,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using OpenTelemetry.Trace;
namespace HordeServer.Tests
{
@@ -52,7 +53,6 @@ namespace HordeServer.Tests
public ServerServiceTest()
{
_pluginCollection = new PluginCollection();
_pluginCollection.Add<AnalyticsPlugin>();
PatchDatadogWriter();
}
@@ -97,6 +97,9 @@ namespace HordeServer.Tests
services.AddLogging(builder => { builder.AddConsole().SetMinimumLevel(LogLevel.Debug); });
services.AddSingleton<IMemoryCache>(sp => new MemoryCache(new MemoryCacheOptions { }));
services.AddSingleton<OpenTelemetry.Trace.Tracer>(sp => TracerProvider.Default.GetTracer("TestTracer"));
services.AddSingleton(sp => new Meter("TestMeter"));
services.AddSingleton(typeof(IAuditLogFactory<>), typeof(AuditLogFactory<>));
services.AddSingleton<ConfigService>();

8
Engine/Source/Programs/Horde/HordeServer.Tests/Ddc/Functional/ContentId/CompressedBlobTests.cs
View File

@@ -41,10 +41,12 @@ namespace HordeServer.Tests.Ddc.FunctionalTests.CompressedBlobs
{
ServerSettings serverSettings = ServiceProvider.GetRequiredService<IOptions<ServerSettings>>().Value;
GlobalConfig globalConfig = ServiceProvider.GetRequiredService<IOptions<GlobalConfig>>().Value;
StorageConfig storageConfig = new StorageConfig();
storageConfig.Backends.Add(new BackendConfig { Id = new BackendId("default"), Type = StorageBackendType.Memory });
storageConfig.Namespaces.Add(new NamespaceConfig { Id = new NamespaceId(TestNamespace), Backend = storageConfig.Backends[^1].Id });
globalConfig.Storage.Backends.Add(new BackendConfig { Id = new BackendId("default"), Type = StorageBackendType.Memory });
globalConfig.Storage.Namespaces.Add(new NamespaceConfig { Id = new NamespaceId(TestNamespace), Backend = globalConfig.Storage.Backends[^1].Id });
GlobalConfig globalConfig = ServiceProvider.GetRequiredService<IOptions<GlobalConfig>>().Value;
globalConfig.Plugins.AddStorageTestConfig(storageConfig);
globalConfig.PostLoad(serverSettings, new List<ILoadedPlugin>());
ConfigService configService = ServiceProvider.GetRequiredService<ConfigService>();

8
Engine/Source/Programs/Horde/HordeServer.Tests/Ddc/Functional/Objects/ReferencesTests.cs
View File

@@ -57,10 +57,12 @@ namespace HordeServer.Tests.Ddc.FunctionalTests.References
ServerSettings serverSettings = ServiceProvider.GetRequiredService<IOptions<ServerSettings>>().Value;
GlobalConfig globalConfig = ServiceProvider.GetRequiredService<IOptions<GlobalConfig>>().Value;
StorageConfig storageConfig = new StorageConfig();
storageConfig.Backends.Add(new BackendConfig { Id = new BackendId("default"), Type = StorageBackendType.Memory });
storageConfig.Namespaces.Add(new NamespaceConfig { Id = TestNamespace, Backend = storageConfig.Backends[^1].Id });
globalConfig.Storage.Backends.Add(new BackendConfig { Id = new BackendId("default"), Type = StorageBackendType.Memory });
globalConfig.Storage.Namespaces.Add(new NamespaceConfig { Id = TestNamespace, Backend = globalConfig.Storage.Backends[^1].Id });
GlobalConfig globalConfig = ServiceProvider.GetRequiredService<IOptions<GlobalConfig>>().Value;
globalConfig.Plugins.AddStorageTestConfig(storageConfig);
globalConfig.PostLoad(serverSettings, new List<ILoadedPlugin>());
ConfigService configService = ServiceProvider.GetRequiredService<ConfigService>();

3
Engine/Source/Programs/Horde/HordeServer.Tests/HordeServer.Tests.csproj
View File

@@ -41,7 +41,8 @@
<Aliases>JobDriver</Aliases>
</ProjectReference>
<ProjectReference Include="..\HordeServer\HordeServer.csproj" />
<ProjectReference Include="..\Plugins\Analytics\HordeServer.Analytics\HordeServer.Analytics.csproj" />
<ProjectReference Include="..\Plugins\Analytics\HordeServer.Analytics.Tests\HordeServer.Analytics.Tests.csproj" />
<ProjectReference Include="..\Plugins\Storage\HordeServer.Storage.Tests\HordeServer.Storage.Tests.csproj" />
</ItemGroup>
<ItemGroup>

9
Engine/Source/Programs/Horde/HordeServer.Tests/Issues/IssueServiceTests.cs
View File

@@ -23,6 +23,7 @@ using HordeServer.Issues;
using HordeServer.Jobs;
using HordeServer.Jobs.Graphs;
using HordeServer.Logs;
using HordeServer.Plugins;
using HordeServer.Projects;
using HordeServer.Server;
using HordeServer.Storage;
@@ -149,11 +150,13 @@ namespace HordeServer.Tests.Issues
projectConfig.Streams.Add(CreateStream(_releaseStreamId, ReleaseStreamName));
projectConfig.Streams.Add(CreateStream(_devStreamId, DevStreamName));
StorageConfig storageConfig = new StorageConfig();
storageConfig.Backends.Add(new BackendConfig { Id = new BackendId("default-backend"), Type = StorageBackendType.Memory });
storageConfig.Namespaces.Add(new NamespaceConfig { Id = new NamespaceId("horde-logs"), Backend = new BackendId("default-backend"), GcDelayHrs = 0.0 });
GlobalConfig globalConfig = new GlobalConfig();
globalConfig.Projects.Add(projectConfig);
globalConfig.Storage.Backends.Add(new BackendConfig { Id = new BackendId("default-backend"), Type = StorageBackendType.Memory });
globalConfig.Storage.Namespaces.Add(new NamespaceConfig { Id = new NamespaceId("horde-logs"), Backend = new BackendId("default-backend"), GcDelayHrs = 0.0 });
globalConfig.Plugins.Add(new PluginName("storage"), storageConfig);
SetConfig(globalConfig);
static StreamConfig CreateStream(StreamId streamId, string streamName)

18
Engine/Source/Programs/Horde/HordeServer.Tests/TestSetup.cs
View File

@@ -117,6 +117,12 @@ namespace HordeServer.Tests
public TestDataController TestDataController => GetTestDataController();
public BisectTasksController BisectTasksController => GetBisectTasksController();
public TestSetup()
{
AddPlugin<AnalyticsPlugin>();
AddPlugin<StoragePlugin>();
}
protected override void ConfigureServices(IServiceCollection services)
{
base.ConfigureServices(services);
@@ -129,9 +135,6 @@ namespace HordeServer.Tests
services.AddSingleton<IAuditLog<AgentId>>(sp => sp.GetRequiredService<IAuditLogFactory<AgentId>>().Create("Agents.Log", "AgentId"));
services.AddSingleton<OpenTelemetry.Trace.Tracer>(sp => TracerProvider.Default.GetTracer("TestTracer"));
services.AddSingleton(sp => new Meter("TestMeter"));
services.AddSingleton<IAccountCollection, AccountCollection>();
services.AddSingleton<IAgentCollection, AgentCollection>();
services.AddSingleton<IAgentTelemetryCollection, AgentTelemetryCollection>();
@@ -199,13 +202,6 @@ namespace HordeServer.Tests
services.AddSingleton<ConformTaskSource>();
services.AddSingleton<ICommitService, CommitService>();
services.AddSingleton<FileObjectStoreFactory>();
services.AddSingleton<IObjectStoreFactory, ObjectStoreFactory>();
services.AddSingleton<StorageService>();
services.AddSingleton<StorageBackendCache>();
services.AddSingleton<BundleCache>();
}
public Task<Fixture> CreateFixtureAsync()
@@ -334,7 +330,7 @@ namespace HordeServer.Tests
{
await agent.TryUpdateWorkspacesAsync(workspaces, false);
}
if (requestShutdown)
{
await agent.TryUpdateAsync(new UpdateAgentOptions { RequestShutdown = true });

12
Engine/Source/Programs/Horde/HordeServer.Tests/Tools/ToolTests.cs
View File

@@ -14,6 +14,7 @@ using HordeServer.Tools;
using HordeCommon;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using HordeServer.Plugins;
namespace HordeServer.Tests.Tools
{
@@ -24,11 +25,14 @@ namespace HordeServer.Tests.Tools
public ToolTests()
{
StorageConfig storageConfig = new StorageConfig();
storageConfig.Backends.Clear();
storageConfig.Backends.Add(new BackendConfig { Id = new BackendId("tools-backend"), Type = StorageBackendType.Memory });
storageConfig.Namespaces.Clear();
storageConfig.Namespaces.Add(new NamespaceConfig { Id = Namespace.Tools, Backend = new BackendId("tools-backend") });
GlobalConfig globalConfig = new GlobalConfig();
globalConfig.Storage.Backends.Clear();
globalConfig.Storage.Backends.Add(new BackendConfig { Id = new BackendId("tools-backend"), Type = StorageBackendType.Memory });
globalConfig.Storage.Namespaces.Clear();
globalConfig.Storage.Namespaces.Add(new NamespaceConfig { Id = Namespace.Tools, Backend = new BackendId("tools-backend") });
globalConfig.Plugins.Add(new PluginName("storage"), storageConfig);
globalConfig.Tools.Add(new ToolConfig(_toolId) { Name = "UnrealGameSync", Description = "Tool for syncing content from source control" });
SetConfig(globalConfig);
}

18
Engine/Source/Programs/Horde/HordeServer.Tests/Tools/ToolsControllerTests.cs
View File

@@ -33,20 +33,24 @@ public class ToolsControllerTests
PluginCollection pluginCollection = new PluginCollection();
pluginCollection.Add<AnalyticsPlugin>();
pluginCollection.Add<StoragePlugin>();
StorageConfig storageConfig = new StorageConfig();
storageConfig.Backends.Clear();
storageConfig.Backends.Add(new BackendConfig { Id = new BackendId("tools-backend"), Type = StorageBackendType.Memory });
storageConfig.Namespaces.Clear();
storageConfig.Namespaces.Add(new NamespaceConfig { Id = Namespace.Tools, Backend = new BackendId("tools-backend") });
GlobalConfig globalConfig = new();
globalConfig.Storage.Backends.Clear();
globalConfig.Storage.Backends.Add(new BackendConfig { Id = new BackendId("tools-backend"), Type = StorageBackendType.Memory });
globalConfig.Storage.Namespaces.Clear();
globalConfig.Storage.Namespaces.Add(new NamespaceConfig { Id = Namespace.Tools, Backend = new BackendId("tools-backend") });
globalConfig.Plugins.AddStorageTestConfig(storageConfig);
globalConfig.Tools.Add(new ToolConfig(toolId) { Name = "Foo", Description = "This is foo", Acl = new AclConfig() { Entries = [aclEntryConfig] }, Public = false });
ServerSettings serverSettings = new() { AuthMethod = AuthMethod.Horde };
globalConfig.PostLoad(serverSettings, pluginCollection.LoadedPlugins);
Dictionary<string, string> settings = new() { { "Horde:AuthMethod", AuthMethod.Horde.ToString() } };
await using FakeHordeWebApp app = new(settings);
ConfigService configService = app.ServiceProvider.GetRequiredService<ConfigService>();
IToolCollection tools = app.ServiceProvider.GetRequiredService<IToolCollection>();
IServiceAccountCollection serviceAccounts = app.ServiceProvider.GetRequiredService<IServiceAccountCollection>();
@@ -55,7 +59,7 @@ public class ToolsControllerTests
List<IUserClaim> claims = [new UserClaim("http://epicgames.com/ue/horde/role", "agent")];
(IServiceAccount _, string token) = await serviceAccounts.CreateAsync(new CreateServiceAccountOptions("myDesc", claims));
// Create tool and deployment
using MemoryStream ms = new(await ToolTests.CreateZipFileDataAsync("foo.txt", "foo content"));
ITool? tool = await tools.GetAsync(toolId);

15
Engine/Source/Programs/Horde/HordeServer/Configuration/ConfigService.cs
View File

@@ -588,6 +588,7 @@ namespace HordeServer.Configuration
JsonObject configObj = await context.PreprocessFileAsync(GetGlobalConfigUri(), configNode, cancellationToken);
CopyJsonNode(configObj, "TelemetryStores", "Plugins.Analytics.Stores");
CopyJsonNode(configObj, "Storage", "Plugins.Storage");
GlobalConfig globalConfig = JsonSerializer.Deserialize<GlobalConfig>(configObj, _jsonOptions)!;
if (globalConfig.VersionEnum < GlobalVersion.Latest)
@@ -624,7 +625,19 @@ namespace HordeServer.Configuration
throw new ConfigException(context, ex.Message, ex);
}
}
/*
static bool TryGetChildNode(JsonNode node, string name, [NotNullWhen(true)] out JsonNode? child)
{
child = node[name];
if (child != null)
{
JsonObject? parentObj = node as JsonObject;
parentObj
parentObj.TryGetPropertyValue(
}
return child != null;
}
*/
static void CopyJsonNode(JsonObject rootObj, string sourcePath, string targetPath)
{
JsonNode? sourceNode = rootObj;

9
Engine/Source/Programs/Horde/HordeServer/Ddc/RequestHelper.cs
View File

@@ -5,7 +5,6 @@ using System.Security.Claims;
using System.Threading.Tasks;
using EpicGames.Horde.Acls;
using EpicGames.Horde.Storage;
using HordeServer.Server;
using HordeServer.Storage;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
@@ -16,12 +15,12 @@ namespace HordeServer.Ddc
class RequestHelper : IRequestHelper
{
readonly IStorageClientFactory _storageClientFactory;
readonly GlobalConfig _globalConfig;
readonly StorageConfig _storageConfig;
public RequestHelper(IStorageClientFactory storageClientFactory, IOptionsSnapshot<GlobalConfig> globalConfig)
public RequestHelper(IStorageClientFactory storageClientFactory, IOptionsSnapshot<StorageConfig> storageConfig)
{
_storageClientFactory = storageClientFactory;
_globalConfig = globalConfig.Value;
_storageConfig = storageConfig.Value;
}
public Task<ActionResult?> HasAccessForGlobalOperationsAsync(ClaimsPrincipal user, AclAction[] aclActions)
@@ -36,7 +35,7 @@ namespace HordeServer.Ddc
{
return Task.FromResult<ActionResult?>(new ForbidResult());
}
if (!_globalConfig.Storage.TryGetNamespace(ns, out NamespaceConfig? namespaceConfig) || aclActions.Any(x => !namespaceConfig.Authorize(x, user)))
if (!_storageConfig.TryGetNamespace(ns, out NamespaceConfig? namespaceConfig) || aclActions.Any(x => !namespaceConfig.Authorize(x, user)))
{
return Task.FromResult<ActionResult?>(new ForbidResult());
}

1
Engine/Source/Programs/Horde/HordeServer/HordeServer.csproj
View File

@@ -135,6 +135,7 @@
<ProjectReference Include="..\..\Shared\EpicGames.Slack\EpicGames.Slack.csproj" />
<ProjectReference Include="..\HordeServer.Shared\HordeServer.Shared.csproj" />
<ProjectReference Include="..\Plugins\Analytics\HordeServer.Analytics\HordeServer.Analytics.csproj" />
<ProjectReference Include="..\Plugins\Storage\HordeServer.Storage\HordeServer.Storage.csproj" />
</ItemGroup>
<ItemGroup>

2
Engine/Source/Programs/Horde/HordeServer/Plugins/PluginCollection.cs
View File

@@ -136,7 +136,7 @@ namespace HordeServer.Plugins
{
if (!_loadedPluginNames.Add(metadata.Name))
{
throw new NotImplementedException();
throw new InvalidOperationException($"An implementation of the {metadata.Name} plugin has already been added");
}
PluginAttribute attr = startupType.GetCustomAttribute<PluginAttribute>()

8
Engine/Source/Programs/Horde/HordeServer/Server/GlobalConfig.cs
View File

@@ -40,7 +40,6 @@ using HordeServer.Perforce;
using HordeServer.Plugins;
using HordeServer.Projects;
using HordeServer.Secrets;
using HordeServer.Storage;
using HordeServer.Streams;
using HordeServer.Tools;
using HordeServer.Utilities;
@@ -229,11 +228,6 @@ namespace HordeServer.Server
/// </summary>
public TimeSpan? AgentShutdownIfDisabledGracePeriod { get; set; } = null;
/// <summary>
/// Storage configuration
/// </summary>
public StorageConfig Storage { get; set; } = new StorageConfig();
/// <summary>
/// Configuration for different artifact types
/// </summary>
@@ -350,8 +344,6 @@ namespace HordeServer.Server
ConfigObject.MergeDefaults<string, PoolConfig>(Pools.Select(x => (x.Id.ToString(), x.Base?.ToString(), x)));
UpdateWorkspacesForPools();
Storage.PostLoad(this);
// Ensure that all plugins have an entry in the global config so they can register their ACLs
foreach (ILoadedPlugin loadedPlugin in loadedPlugins)
{

18
Engine/Source/Programs/Horde/HordeServer/ServerSettings.cs
View File

@@ -16,7 +16,6 @@ using EpicGames.Horde.Tools;
using EpicGames.Perforce;
using HordeServer.Agents.Fleet;
using HordeServer.Server;
using HordeServer.Storage;
using HordeServer.Tools;
namespace HordeServer
@@ -502,7 +501,7 @@ namespace HordeServer
/// <summary>
/// Overridden settings for storage backends. Useful for running against a production server with custom backends.
/// </summary>
public List<BackendConfig> StorageBackends { get; set; } = new List<BackendConfig>();
public string LogServiceWriteCacheType { get; set; } = "InMemory";
/// <summary>
/// Whether to log json to stdout
@@ -713,21 +712,6 @@ namespace HordeServer
/// </summary>
public bool OpenBrowser { get; set; } = false;
/// <summary>
/// Directory to use for the coarse-grained backend cache. This caches full bundles downloaded from the upstream object store.
/// </summary>
public string? BundleCacheDir { get; set; }
/// <summary>
/// Maximum size of the storage cache on disk. Accepts standard binary suffixes (kb, mb, gb, tb, etc...)
/// </summary>
public string BundleCacheSize { get; set; } = "1gb";
/// <summary>
/// Accessor for the bundle cache size in bytes
/// </summary>
public long BundleCacheSizeBytes => StringUtils.ParseBytesString(BundleCacheSize);
/// <summary>
/// Directory to store the fine-grained block cache. This caches individual exports embedded in bundles.
/// </summary>

21
Engine/Source/Programs/Horde/HordeServer/Startup.cs
View File

@@ -27,8 +27,6 @@ using EpicGames.Horde.Logs;
using EpicGames.Horde.Projects;
using EpicGames.Horde.Server;
using EpicGames.Horde.Storage;
using EpicGames.Horde.Storage.Bundles;
using EpicGames.Horde.Storage.ObjectStores;
using EpicGames.Horde.Streams;
using EpicGames.Horde.Users;
using EpicGames.Redis;
@@ -77,7 +75,6 @@ using HordeServer.Server;
using HordeServer.Server.Notices;
using HordeServer.ServiceAccounts;
using HordeServer.Storage;
using HordeServer.Storage.ObjectStores;
using HordeServer.Streams;
using HordeServer.Tasks;
using HordeServer.Tools;
@@ -319,6 +316,7 @@ namespace HordeServer
// Register the plugin collection
PluginCollection pluginCollection = new PluginCollection();
pluginCollection.Add<AnalyticsPlugin>();
pluginCollection.Add<StoragePlugin>();
services.AddSingleton<IPluginCollection>(pluginCollection);
// Register all the plugin services
@@ -506,13 +504,8 @@ namespace HordeServer
services.AddSingleton<DeviceService>();
services.AddSingleton<NoticeService>();
services.AddSingleton<StorageService>();
services.AddSingleton<IStorageService>(sp => sp.GetRequiredService<StorageService>());
services.AddScoped(sp => sp.GetRequiredService<StorageService>().CreateStorageClientFactory(sp.GetRequiredService<IOptionsSnapshot<GlobalConfig>>().Value));
services.AddSingleton<IBlockCache>(sp => CreateBlockCache(sp));
services.AddSingleton<TestDataService>();
services.AddSingleton<BundleCache>();
services.AddSingleton<StorageBackendCache>(CreateStorageBackendCache);
if (settings.JiraUrl != null)
{
@@ -539,11 +532,6 @@ namespace HordeServer
services.AddScoped<ISecretCollection, SecretCollection>();
services.AddSingleton<ISecretProvider, AwsParameterStoreSecretProvider>();
// Storage providers
services.AddSingleton<IObjectStoreFactory, ObjectStoreFactory>();
services.AddSingleton<AwsObjectStoreFactory>();
services.AddSingleton<FileObjectStoreFactory>();
if (settings.WithAws)
{
AWSOptions awsOptions = Configuration.GetAWSOptions();
@@ -912,13 +900,6 @@ namespace HordeServer
return BlockCache.Create(cacheDir, (int)(serverSettings.BlockCacheSizeBytes / (1024 * 1024 * 1024)));
}
static StorageBackendCache CreateStorageBackendCache(IServiceProvider serviceProvider)
{
ServerSettings serverSettings = serviceProvider.GetRequiredService<IOptions<ServerSettings>>().Value;
DirectoryReference cacheDir = DirectoryReference.Combine(ServerApp.DataDir, String.IsNullOrEmpty(serverSettings.BundleCacheDir) ? "Cache" : serverSettings.BundleCacheDir);
return new StorageBackendCache(cacheDir, serverSettings.BundleCacheSizeBytes, serviceProvider.GetRequiredService<ILogger<StorageBackendCache>>());
}
public static void ConfigureFormatters()
{
LogValueFormatter.RegisterTypeAnnotation<AgentId>("AgentId");

7
Engine/Source/Programs/Horde/HordeServer/appsettings.json
View File

@@ -3,7 +3,12 @@
"ConsoleLogLevel": "Debug",
"DashboardUrl": "https://localhost:3000",
"CorsEnabled": true,
"CorsOrigin": "https://localhost:5003;http://localhost:3000"
"CorsOrigin": "https://localhost:5003;http://localhost:3000",
"Plugins": {
"analytics": {
"Foo": 1234
}
}
},
"Serilog": {
"MinimumLevel": {

21
Engine/Source/Programs/Horde/Plugins/Analytics/HordeServer.Analytics.Tests/AnalyticsTestExtensions.cs Normal file
View File

@@ -0,0 +1,21 @@
// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using HordeServer.Plugins;
using HordeServer.Telemetry;
using HordeServer.Telemetry.Metrics;
using HordeServer.Telemetry.Sinks;
using Microsoft.Extensions.DependencyInjection;
namespace HordeServer.Analytics.Tests
{
public static class AnalyticsTestExtensions
{
public static void AddAnalyticsTestConfig(this PluginConfigCollection configCollection, AnalyticsConfig config)
=> configCollection[new PluginName("Analytics")] = config;
}
}

Some files were not shown because too many files have changed in this diff Show More