Horde: Add AgentStatus to GetAgentResponse.

#jira UE-208261
#rnx

[CL 31884787 by ben marsh in ue5-main branch]
This commit is contained in:
ben marsh
2024-02-28 16:18:11 -05:00
parent 2d1d53be8e
commit 2d644d7f34
14 changed files with 183 additions and 145 deletions

View File

@@ -107244,7 +107244,7 @@
<File Name="Engine/Source/Programs/Horde/Horde.Server/Properties/PublishProfiles/FolderProfile.pubxml" Hash="bed5b0353dfa55d764e071b14f4fc939fa1bdbfb" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Properties/PublishProfiles/FolderProfile.pubxml.user" Hash="b55f89829f744d6a8c8649a0aaec7f2d1c9a6017" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/grpc/health/v1/health.proto" Hash="aa2d40f8d7b51ec556fd1e47c751ab9deb277782" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/common/agent_status.proto" Hash="b2a2674f8baa455ca4890a38fe60c4cec1103903" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/common/agent_status.proto" Hash="8ed0dd8fd80725e718068fbfa436e8b62b7e56bd" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/common/event_severity.proto" Hash="eb251918223512ba339eba810f9c884b096f819b" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/common/io_hash.proto" Hash="5b46cbe2d93702bc4f8b5d1947d33e48ef226ae2" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/common/jobstep_outcome.proto" Hash="ef858453aaf8f87c630569cb7462557fa9770587" />
@@ -107255,7 +107255,7 @@
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/common/namespace_id.proto" Hash="02c44a07e3eb84aa1ace69e3bc239672ed1804cc" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/common/priority.proto" Hash="302451001483c7bf17048d1dbb80f420432b6ed5" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/horde_rpc.proto" Hash="8bad2fef6e074c9175e8d6611317e0ddf06d689d" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/horde_rpc_messages.proto" Hash="a19907a3c9d60d071d950be855617031f9af26d6" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/horde_rpc_messages.proto" Hash="b4796260ad195b057212a01fb69ea072035d48f4" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/job_rpc.proto" Hash="95d124a6945b1ec2276ba6e1919c88c103169e02" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/job_rpc_messages.proto" Hash="04373b7418f723e298937981f0a04dcdf2c0c768" />
<File Name="Engine/Source/Programs/Horde/Horde.Server/Protos/horde/log_rpc.proto" Hash="191b8f0fbc01769b3ac39d734b5b751db389223a" />
@@ -166965,6 +166965,7 @@
<Blob Hash="8ecd1da73a4b0ae43a9b27f03ea61f65342177ef" Size="15204" PackHash="87f9081b839dd50b50677f943504b49c75f21983" PackOffset="153576" />
<Blob Hash="8ecd2ba4a02e89f92395b6f30f3e980d3adab0ca" Size="74238" PackHash="1b309a1386b9b9e9e7be8a2037d48ffa0a36db53" PackOffset="660819" />
<Blob Hash="8ed05ecc185bd23ddaa3a08c1b3ad881b6c7bc85" Size="30524" PackHash="e5cd49b94c36c20503b460c17c91e7794e2e832c" PackOffset="196406" />
<Blob Hash="8ed0dd8fd80725e718068fbfa436e8b62b7e56bd" Size="1317" PackHash="8000930610244b350287cd930dddc5556beac913" PackOffset="8" />
<Blob Hash="8ed22f51a9f0a21aaa7d4bc0088aca4e8da30877" Size="6457" PackHash="9a9e37235eb40f21cdd2942e2f2f358971fab9d5" PackOffset="1748179" />
<Blob Hash="8ed61adfa593e4d5a34ebecae0177446835e259d" Size="359" PackHash="22c1ca1b28cefe888df197da58df6fe961b364dd" PackOffset="1427199" />
<Blob Hash="8ed729de927faa30d2791d177955477eb0c2cc24" Size="84639" PackHash="84b9afbc4cb09760e5fb4b1f84e78705968bce01" PackOffset="64633" />
@@ -172251,7 +172252,6 @@
<Blob Hash="a197c3c55530e0d2752222dc586368e7de8c8c65" Size="74" PackHash="830ccc134f8ffeba50f1db0095689c5e507fe906" PackOffset="298675" />
<Blob Hash="a197d413ce9db56e3132cac61a89b9c5cb27a2fd" Size="5156" PackHash="48d24e2982f6d98825baced1b4ff4d71c7aecf7a" PackOffset="1287032" />
<Blob Hash="a198ba15a3d6b962968450d822e90243d7832bc7" Size="3371916" PackHash="739e0bd07807483987ca51619fd0f4419a4270a6" PackOffset="8" />
<Blob Hash="a19907a3c9d60d071d950be855617031f9af26d6" Size="13814" PackHash="4d949c48d23f1f7e7620e0179e1fd360f2efc6e9" PackOffset="1872151" />
<Blob Hash="a1996310150b0d9c022a82097245044650039ebc" Size="671576" PackHash="475ed761d6cbeaaddbe2086955dde4296a5aac66" PackOffset="1127866" />
<Blob Hash="a1997367a925cd3e2acb7dcb42fcb8cff6116b90" Size="493696" PackHash="ab48571503ec58eefe9029be85c6b950e6dd3d76" PackOffset="69189" />
<Blob Hash="a1997aa2fc75314b2fe62b2461fb6d4c38be8f46" Size="11798" PackHash="d314351c919854cfd20c02d3edd3ea1c8c7ef33c" PackOffset="8" />
@@ -177025,7 +177025,6 @@
<Blob Hash="b29d38827e69c1df187dc773ceb9c9fd030578d7" Size="6663" PackHash="908da82afd0d4e2fe6f9aa1803f1b0f94bd8cd26" PackOffset="20283" />
<Blob Hash="b29e0cf7422ed50842e87bba6479c6b959f3f5e5" Size="269824" PackHash="301b6d535d22d6aa6615be0620208ed50b06be83" PackOffset="8" />
<Blob Hash="b29fb3cec5766c912b4dbda3831c4f3baabbc75f" Size="20068" PackHash="afb74e22842bf990b4eddd5318047522e2647cbb" PackOffset="8" />
<Blob Hash="b2a2674f8baa455ca4890a38fe60c4cec1103903" Size="1310" PackHash="41b316be12c9ef69a1271ceb32552e618dc0400d" PackOffset="8" />
<Blob Hash="b2a4045b1c89fe3f55aa8af5c01ff19469d1d705" Size="30655" PackHash="ad79485db78dd448bb1f02e52dcd035ee6c08b63" PackOffset="1521489" />
<Blob Hash="b2a49c16d418c951a0876404c101dc3d6297ef41" Size="353" PackHash="4fd5e6b3e5968fbc7d109a13f1a108502536b346" PackOffset="188122" />
<Blob Hash="b2a586e84d540d9080bb9f711721bd5ecdbb074e" Size="742400" PackHash="b591e3e88dfeee978de727da24552e0e99dcf585" PackOffset="994208" />
@@ -177545,6 +177544,7 @@
<Blob Hash="b478744d6df1a0b6fc1308b0ba087ea649779916" Size="16136" PackHash="44138a714bbe56603e10fd14f9684f39ffe78a6b" PackOffset="1099466" />
<Blob Hash="b478e4b5466c68145a7289fc2c28a14c86b2842a" Size="13342" PackHash="b82dede9842ce814fdf1bc109f359aad41fdaa7e" PackOffset="257685" />
<Blob Hash="b478ffc172b2e4daf7ee96d59d07de6b0a935f60" Size="546656" PackHash="cca95b0136f665c2778b1f0370fd6733dd954b6e" PackOffset="1549528" />
<Blob Hash="b4796260ad195b057212a01fb69ea072035d48f4" Size="13823" PackHash="8000930610244b350287cd930dddc5556beac913" PackOffset="1325" />
<Blob Hash="b47dd464ee2ad12a7e5167692ac1c47193f9dcfc" Size="576" PackHash="60b2893742f3204997fa262ba1acf3bc51e2f1ab" PackOffset="1105" />
<Blob Hash="b47f5438eef2e6c4d7b2391beaa6f3eb27b31b33" Size="305" PackHash="c88e257c0a2e9ba0049babe50374dd7c3c3b7fbe" PackOffset="2076574" />
<Blob Hash="b48095f579eb210d143c8d416c5480c4fdfbef6d" Size="47616" PackHash="0b73681e2ee512a09ccd8ba00902e5fa7755bed1" PackOffset="1445896" />
@@ -201263,7 +201263,6 @@
<Pack Hash="41a32a97cf1f890fea4d62da262cf5888a92e81d" Size="1892756" CompressedSize="1797617" RemotePath="UnrealEngine-25328963" />
<Pack Hash="41a4ea33a14b50029b65b19bf051fe16dafeb391" Size="1807" CompressedSize="1837" RemotePath="UnrealEngine-25328963" />
<Pack Hash="41a64ce6e0a1bbd3670bb2c57977927b1030a97f" Size="2555192" CompressedSize="934652" RemotePath="UnrealEngine-25357016" />
<Pack Hash="41b316be12c9ef69a1271ceb32552e618dc0400d" Size="1318" CompressedSize="751" RemotePath="UnrealEngine-31872174" />
<Pack Hash="41b3e02e478a8281ce1931e62acdd97574623236" Size="1346420" CompressedSize="519563" RemotePath="UnrealEngine-25328963" />
<Pack Hash="41b5319cb97138e7ae09626f53251fc2ed3ca51b" Size="14184636" CompressedSize="12556057" RemotePath="UnrealEngine-31211930" />
<Pack Hash="41b99a1c97edf8bf62e758375a51e1b5079f5682" Size="39776" CompressedSize="4387" RemotePath="UnrealEngine-25357016" />
@@ -203253,6 +203252,7 @@
<Pack Hash="7fe5ca78a301242de6ca9e571139bd06a4d2af70" Size="2076497" CompressedSize="379664" RemotePath="UnrealEngine-31562413" />
<Pack Hash="7fe9b5ec8a7c2b4a98d46991db83291048f47de7" Size="1265180" CompressedSize="383020" RemotePath="UnrealEngine-25328963" />
<Pack Hash="7ff615b17512f581a231cdbca8c54337ce6d4c77" Size="1640504" CompressedSize="658284" RemotePath="UnrealEngine-25379944" />
<Pack Hash="8000930610244b350287cd930dddc5556beac913" Size="15148" CompressedSize="3991" RemotePath="UnrealEngine-31884787" />
<Pack Hash="801617cf2a66385a2ce3b78019507c39bb0631be" Size="1864968" CompressedSize="798440" RemotePath="UnrealEngine-25357016" />
<Pack Hash="8017bf2fda250adce86f10dbb805484553d6a800" Size="1092610" CompressedSize="347441" RemotePath="UnrealEngine-29536258" />
<Pack Hash="80216614d703972b5174b07a4dbdfb350bfe87cc" Size="10272102" CompressedSize="9971326" RemotePath="UnrealEngine-31117086" />

View File

@@ -7,7 +7,6 @@ using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Horde.Agent.Services;
using Horde.Agent.Utility;
using HordeCommon;
using HordeCommon.Rpc;
using HordeCommon.Rpc.Messages;
using Microsoft.Extensions.DependencyInjection;
@@ -281,19 +280,19 @@ namespace Horde.Agent.Leases
bool busy = _statusService.IsBusy;
if (stopping)
{
updateSessionRequest.Status = AgentStatus.Stopping;
updateSessionRequest.Status = RpcAgentStatus.Stopping;
}
else if (_unhealthy)
{
updateSessionRequest.Status = AgentStatus.Unhealthy;
updateSessionRequest.Status = RpcAgentStatus.Unhealthy;
}
else if (busy)
{
updateSessionRequest.Status = AgentStatus.Busy;
updateSessionRequest.Status = RpcAgentStatus.Busy;
}
else
{
updateSessionRequest.Status = AgentStatus.Ok;
updateSessionRequest.Status = RpcAgentStatus.Ok;
}
// Update the capabilities whenever the background task has generated a new instance
@@ -360,7 +359,7 @@ namespace Horde.Agent.Leases
}
// Update the session result if we've transitioned to stopped
if (updateSessionResponse.Status == AgentStatus.Stopped)
if (updateSessionResponse.Status == RpcAgentStatus.Stopped)
{
_logger.LogInformation("Agent status is stopped; returning from session update loop.");
return _sessionResult ?? new SessionResult(SessionOutcome.BackOff);

View File

@@ -10,7 +10,6 @@ using Grpc.Core;
using Grpc.Net.Client;
using Horde.Agent.Utility;
using Horde.Common.Rpc;
using HordeCommon;
using HordeCommon.Rpc;
using HordeCommon.Rpc.Messages;
using Microsoft.Extensions.Logging;
@@ -199,7 +198,7 @@ namespace Horde.Agent.Services
// Create the session information
CreateSessionRequest sessionRequest = new CreateSessionRequest();
sessionRequest.Id = registrationInfo.Id;
sessionRequest.Status = AgentStatus.Ok;
sessionRequest.Status = RpcAgentStatus.Ok;
sessionRequest.Capabilities = capabilities;
sessionRequest.Version = AgentApp.Version;
@@ -364,7 +363,7 @@ namespace Horde.Agent.Services
string fileName = imageFile.GetFileName();
if (_processNamesToTerminate.TryGetValue(fileName, out TerminateCondition terminateFlags))
{
if(terminateFlags == TerminateCondition.None || (terminateFlags & condition) != 0)
if (terminateFlags == TerminateCondition.None || (terminateFlags & condition) != 0)
{
return true;
}

View File

@@ -3,12 +3,12 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Horde.Server.Agents;
using Horde.Server.Agents.Sessions;
using HordeCommon;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using EpicGames.Horde.Agents;
using EpicGames.Horde.Agents.Leases;
using EpicGames.Horde.Agents.Pools;
using Horde.Server.Agents;
using Horde.Server.Agents.Sessions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
namespace Horde.Server.Tests.Agents;
@@ -20,7 +20,7 @@ public class AgentCollectionTests : TestSetup
private readonly AgentLease _lease2;
private readonly AgentLease _leaseWithParent3;
private readonly AgentLease _leaseWithParent4;
public AgentCollectionTests()
{
_agent = CreateAgentAsync(new PoolId("pool1"), ephemeral: true).Result;
@@ -33,12 +33,12 @@ public class AgentCollectionTests : TestSetup
[TestMethod]
public async Task AddLeaseAsync()
{
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await UpdateAgentAsync();
await AgentCollection.TryAddLeaseAsync(_agent, _leaseWithParent3);
await AgentCollection.TryAddLeaseAsync(_agent, _leaseWithParent3);
await UpdateAgentAsync();
await AgentCollection.TryAddLeaseAsync(_agent, _leaseWithParent4);
await UpdateAgentAsync();
@@ -49,81 +49,81 @@ public class AgentCollectionTests : TestSetup
Assert.IsTrue(leases.Contains(_leaseWithParent3.Id));
Assert.IsTrue(leases.Contains(_leaseWithParent4.Id));
}
[TestMethod]
public async Task StartSessionAsync()
{
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await UpdateAgentAsync();
await AgentCollection.TryAddLeaseAsync(_agent, _lease2);
await AgentCollection.TryAddLeaseAsync(_agent, _lease2);
await UpdateAgentAsync();
await AgentCollection.TryStartSessionAsync(_agent, SessionIdUtils.GenerateNewId(), DateTime.UtcNow, AgentStatus.Ok,
await AgentCollection.TryStartSessionAsync(_agent, SessionIdUtils.GenerateNewId(), DateTime.UtcNow, AgentStatus.Ok,
new List<string>(), new Dictionary<string, int>(), new List<PoolId>(), new List<PoolId>(), DateTime.UtcNow, null);
List<LeaseId> leases = await AgentCollection.FindActiveLeaseIdsAsync();
Assert.AreEqual(0, leases.Count);
}
[TestMethod]
public async Task UpdateSession_WithEmptyLeasesAsync()
{
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await UpdateAgentAsync();
await AgentCollection.TryAddLeaseAsync(_agent, _lease2);
await AgentCollection.TryAddLeaseAsync(_agent, _lease2);
await UpdateAgentAsync();
await AgentCollection.TryUpdateSessionAsync(_agent, null, null, null, null, null, new List<AgentLease>());
List<LeaseId> leases = await AgentCollection.FindActiveLeaseIdsAsync();
Assert.AreEqual(0, leases.Count);
}
[TestMethod]
public async Task UpdateSession_WithNewLeasesAsync()
{
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await UpdateAgentAsync();
await AgentCollection.TryUpdateSessionAsync(_agent, null, null, null, null, null, new List<AgentLease> { _lease1, _lease2 });
List<LeaseId> leases = await AgentCollection.FindActiveLeaseIdsAsync();
Assert.AreEqual(2, leases.Count);
Assert.IsTrue(leases.Contains(_lease1.Id));
Assert.IsTrue(leases.Contains(_lease2.Id));
}
[TestMethod]
public async Task UpdateSession_WithOneLeaseRemovedAsync()
{
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await UpdateAgentAsync();
await AgentCollection.TryAddLeaseAsync(_agent, _lease2);
await AgentCollection.TryAddLeaseAsync(_agent, _lease2);
await UpdateAgentAsync();
await AgentCollection.TryUpdateSessionAsync(_agent, null, null, null, null, null, new List<AgentLease> { _lease1 });
List<LeaseId> leases = await AgentCollection.FindActiveLeaseIdsAsync();
Assert.AreEqual(1, leases.Count);
Assert.IsTrue(leases.Contains(_lease1.Id));
}
[TestMethod]
public async Task CancelLeaseAsync()
{
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await UpdateAgentAsync();
await AgentCollection.TryAddLeaseAsync(_agent, _lease2);
await AgentCollection.TryAddLeaseAsync(_agent, _lease2);
await UpdateAgentAsync();
await AgentCollection.TryCancelLeaseAsync(_agent, 0);
List<LeaseId> leases = await AgentCollection.FindActiveLeaseIdsAsync();
@@ -131,32 +131,32 @@ public class AgentCollectionTests : TestSetup
Assert.AreEqual(1, leases.Count);
Assert.IsTrue(leases.Contains(_lease2.Id));
}
[TestMethod]
public async Task TerminateSessionAsync()
{
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await UpdateAgentAsync();
await AgentCollection.TryAddLeaseAsync(_agent, _lease2);
await AgentCollection.TryAddLeaseAsync(_agent, _lease2);
await UpdateAgentAsync();
await AgentCollection.TryTerminateSessionAsync(_agent);
List<LeaseId> leases = await AgentCollection.FindActiveLeaseIdsAsync();
Assert.AreEqual(0, leases.Count);
}
[TestMethod]
public async Task GetChildLeaseIdsAsync()
{
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await AgentCollection.TryAddLeaseAsync(_agent, _lease1);
await UpdateAgentAsync();
await AgentCollection.TryAddLeaseAsync(_agent, _leaseWithParent3);
await AgentCollection.TryAddLeaseAsync(_agent, _leaseWithParent3);
await UpdateAgentAsync();
await AgentCollection.TryAddLeaseAsync(_agent, _leaseWithParent4);
await UpdateAgentAsync();

View File

@@ -14,7 +14,6 @@ using Horde.Server.Auditing;
using Horde.Server.Jobs;
using Horde.Server.Server;
using Horde.Server.Utilities;
using HordeCommon;
using HordeCommon.Rpc.Messages;
using Microsoft.AspNetCore.Mvc;
using Microsoft.VisualStudio.TestTools.UnitTesting;
@@ -60,12 +59,12 @@ public class AgentServiceTest : TestSetup
Assert.IsFalse(AgentService.AuthorizeSession(agent, GetUser(agent), out string reason));
Assert.IsTrue(reason.Contains("expired", StringComparison.Ordinal));
}
private static long ToUnixTime(DateTime dateTime)
{
return new DateTimeOffset(dateTime).ToUnixTimeSeconds();
}
[TestMethod]
public async Task LastStatusChangeDuringSessionCreateAsync()
{
@@ -73,15 +72,15 @@ public class AgentServiceTest : TestSetup
IAgent agent = await AgentService.CreateAgentAsync("agent1", true, null);
Assert.AreEqual(AgentStatus.Unspecified, agent.Status);
Assert.IsFalse(agent.LastStatusChange.HasValue);
// A session has been created, status change timestamp is current time
agent = await AgentService.CreateSessionAsync(agent, AgentStatus.Ok, new List<string>(), new Dictionary<string, int>(), "v1");
Assert.AreEqual(AgentStatus.Ok, agent.Status);
Assert.AreEqual(ToUnixTime(Clock.UtcNow), ToUnixTime(agent.LastStatusChange!.Value));
DateTime lastStatusChange = agent.LastStatusChange!.Value;
await Clock.AdvanceAsync(TimeSpan.FromMinutes(1));
// The session is re-created, status change timestamp is same as when it first got created
agent = await AgentService.CreateSessionAsync(agent, AgentStatus.Ok, new List<string>(), new Dictionary<string, int>(), "v1");
Assert.AreEqual(AgentStatus.Ok, agent.Status);
@@ -101,7 +100,7 @@ public class AgentServiceTest : TestSetup
// When saved as MongoDB documents, some precision is lost. This compares only Unix seconds.
Assert.AreEqual(ToUnixTime(expected), ToUnixTime(actual!.Value));
}
private static void AssertNotEqual(DateTime expected, DateTime? actual)
{
// When saved as MongoDB documents, some precision is lost. This compares only Unix seconds.
@@ -130,7 +129,7 @@ public class AgentServiceTest : TestSetup
AssertEqual(lastStatusChange, agent.LastStatusChange);
}
}
[TestMethod]
public async Task AuditLogAwsInstanceTypeChangesAsync()
{
@@ -142,16 +141,16 @@ public class AgentServiceTest : TestSetup
await agentLogger.FlushAsync();
return await agentLogger.FindAsync().AnyAsync(x => x.Data.Contains(text, StringComparison.Ordinal));
}
List<string> props = new () { $"{KnownPropertyNames.AwsInstanceType}=m5.large" };
List<string> props = new() { $"{KnownPropertyNames.AwsInstanceType}=m5.large" };
IAgent agent = await AgentService.CreateSessionAsync(fixture.Agent1, AgentStatus.Ok, props, new Dictionary<string, int>(), "test");
Assert.IsFalse(await AuditLogContains("AWS EC2 instance type changed"));
props = new () { $"{KnownPropertyNames.AwsInstanceType}=c6.xlarge" };
props = new() { $"{KnownPropertyNames.AwsInstanceType}=c6.xlarge" };
agent = await AgentService.CreateSessionAsync(agent, AgentStatus.Ok, props, new Dictionary<string, int>(), "test");
Assert.IsTrue(await AuditLogContains("AWS EC2 instance type changed"));
}
[TestMethod]
public async Task GetAgentRateTestAsync()
{
@@ -159,27 +158,27 @@ public class AgentServiceTest : TestSetup
IAgent agent2 = await AgentService.CreateAgentAsync("agent2", true, null);
await AgentService.CreateSessionAsync(agent1, AgentStatus.Ok, new List<string>() { "aws-instance-type=c5.24xlarge", "osfamily=windows" }, new Dictionary<string, int>(), "test");
await AgentService.CreateSessionAsync(agent2, AgentStatus.Ok, new List<string>() { "aws-instance-type=c4.4xLARge", "osfamily=WinDowS" }, new Dictionary<string, int>(), "test");
List<AgentRateConfig> agentRateConfigs = new()
{
new AgentRateConfig() { Condition = "aws-instance-type == 'c5.24xlarge' && osfamily == 'windows'", Rate = 200, },
new AgentRateConfig() { Condition = "aws-instance-type == 'c4.4xlarge' && osfamily == 'windows'", Rate = 300 }
};
await AgentService.UpdateRateTableAsync(agentRateConfigs);
double? rate1 = await AgentService.GetRateAsync(agent1.Id);
Assert.AreEqual(200, rate1!.Value, 0.1);
double? rate2 = await AgentService.GetRateAsync(agent2.Id);
Assert.AreEqual(300, rate2!.Value, 0.1);
}
[TestMethod]
public async Task EphemeralTestAsync()
{
IAgent agent = await CreateAgentAsync(new PoolId("pool1"), ephemeral: true);
Assert.IsTrue(agent.Ephemeral);
agent = (await AgentService.GetAgentAsync(agent.Id))!;
Assert.IsTrue(agent.Ephemeral);
Assert.AreEqual(AgentStatus.Ok, agent.Status);
@@ -187,13 +186,13 @@ public class AgentServiceTest : TestSetup
// Let background task run for purging outdated sessions, which will terminate session for our agent
await Clock.AdvanceAsync(TimeSpan.FromHours(1));
await AgentService.TickAsync(CancellationToken.None);
// Ephemeral agent is marked as deleted once its session is terminated
Assert.IsTrue((await AgentService.GetAgentAsync(agent.Id))!.Deleted);
await Clock.AdvanceAsync(TimeSpan.FromDays(8));
await AgentService.TickAsync(CancellationToken.None);
// Once more time has passed, the ephemeral agent marked as deleted is removed from database
Assert.IsNull(await AgentService.GetAgentAsync(agent.Id));
}

View File

@@ -9,16 +9,16 @@ using System.Threading;
using System.Threading.Tasks;
using Amazon.AutoScaling;
using Amazon.AutoScaling.Model;
using EpicGames.Horde.Agents;
using EpicGames.Horde.Agents.Leases;
using EpicGames.Horde.Agents.Pools;
using Horde.Server.Agents;
using Horde.Server.Agents.Fleet;
using Horde.Server.Utilities;
using HordeCommon;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
using EpicGames.Horde.Agents.Leases;
using EpicGames.Horde.Agents.Pools;
namespace Horde.Server.Tests.Fleet;
@@ -61,13 +61,13 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
_asgMock
.Setup(x => x.CompleteLifecycleActionAsync(It.IsAny<CompleteLifecycleActionRequest>(), It.IsAny<CancellationToken>()))
.Returns(OnCompleteLifecycleActionAsync);
ServerSettings ss = new () { AwsAutoScalingQueueUrls = new [] { _queueUrl1, _queueUrl2 } };
ServerSettings ss = new() { AwsAutoScalingQueueUrls = new[] { _queueUrl1, _queueUrl2 } };
_asgLifecycleService = new AwsAutoScalingLifecycleService(AgentService, GetRedisServiceSingleton(), AgentCollection, Clock, new TestOptionsMonitor<ServerSettings>(ss), ServiceProvider, Tracer, logger);
_asgLifecycleService.SetAmazonClientsTesting(_asgMock.Object, _fakeSqs);
await _asgLifecycleService.StartAsync(CancellationToken.None);
}
[TestMethod]
public async Task TerminationRequested_WithAgentInService_MarkedForShutdownAsync()
{
@@ -75,7 +75,7 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
LifecycleActionEvent lae = new() { Ec2InstanceId = "i-1234", LifecycleActionToken = "action-token-test", Origin = "AutoScalingGroup" };
IAgent agent = await CreateAgentAsync(new PoolId("pool1"), properties: new() { KnownPropertyNames.AwsInstanceId + "=" + lae.Ec2InstanceId }, ephemeral: true);
await AgentService.DeleteAgentAsync(agent);
// Act
await _fakeSqs.SendMessageAsync(_queueUrl1, JsonSerializer.Serialize(lae));
await _asgLifecycleService.ReceiveLifecycleEventsAsync(_queueUrl1, CancellationToken.None);
@@ -84,7 +84,7 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
agent = (await AgentService.GetAgentAsync(agent.Id))!;
Assert.IsTrue(agent.RequestShutdown);
}
[TestMethod]
public async Task TerminationRequested_WithAgentOnline_LifecycleIsContinuedAsync()
{
@@ -92,14 +92,14 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
LifecycleActionEvent lae = new() { Ec2InstanceId = "i-1234", LifecycleActionToken = "action-token-test", Origin = "AutoScalingGroup" };
await CreateAgentAsync(new PoolId("pool1"), properties: new() { KnownPropertyNames.AwsInstanceId + "=" + lae.Ec2InstanceId });
await _asgLifecycleService.InitiateTerminationAsync(lae, CancellationToken.None);
// Act
await Clock.AdvanceAsync(_asgLifecycleService.LifecycleUpdaterInterval + TimeSpan.FromSeconds(10));
// Assert
AssertLifecycleUpdate(lae, AwsAutoScalingLifecycleService.ActionContinue, _lifecycleUpdates[0]);
}
[TestMethod]
public async Task TerminationRequested_WithAgentGoingFromOnlineToOfflineAsync()
{
@@ -110,27 +110,27 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
agent = await AgentService.CreateSessionAsync(agent, AgentStatus.Ok, props, new Dictionary<string, int>(), "v1");
await _asgLifecycleService.InitiateTerminationAsync(lae, CancellationToken.None);
TimeSpan extraMargin = TimeSpan.FromSeconds(10);
// Agent is online (e.g still running a job) and the lifecycle update tick is triggered
await Clock.AdvanceAsync(_asgLifecycleService.LifecycleUpdaterInterval + extraMargin);
AssertLifecycleUpdate(lae, AwsAutoScalingLifecycleService.ActionContinue, _lifecycleUpdates[0]);
// Agent still online
await Clock.AdvanceAsync(_asgLifecycleService.LifecycleUpdaterInterval + extraMargin);
AssertLifecycleUpdate(lae, AwsAutoScalingLifecycleService.ActionContinue, _lifecycleUpdates[1]);
// Agent responded to shutdown request, now safe to notify AWS ASG the termination can take place
agent = await AgentService.CreateSessionAsync(agent, AgentStatus.Stopped, props, new Dictionary<string, int>(), "v1");
await Clock.AdvanceAsync(_asgLifecycleService.LifecycleUpdaterInterval + extraMargin);
AssertLifecycleUpdate(lae, AwsAutoScalingLifecycleService.ActionAbandon, _lifecycleUpdates[2]);
// Trigger lifecycle update tick once again and no further updates should have been sent
await Clock.AdvanceAsync(_asgLifecycleService.LifecycleUpdaterInterval + extraMargin);
Assert.AreEqual(3, _lifecycleUpdates.Count);
_ = agent;
}
[TestMethod]
public async Task TerminationRequested_WithAgentInWarmPool_LifecycleIsAbandonedAsync()
{
@@ -139,11 +139,11 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
// Act
await _asgLifecycleService.InitiateTerminationAsync(lae, CancellationToken.None);
// Assert
AssertLifecycleUpdate(lae, AwsAutoScalingLifecycleService.ActionAbandon, _lifecycleUpdates[0]);
}
[TestMethod]
public async Task GetInstancesAvailableForTermination_IdleAgent_ReturnsInstanceIdAsync()
{
@@ -155,9 +155,9 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
List<string> instanceIds = await _asgLifecycleService.GetInstancesAvailableForTerminationAsync(e, CancellationToken.None);
// Assert
CollectionAssert.AreEqual(new List<string> () { "i-1000"}, instanceIds);
CollectionAssert.AreEqual(new List<string>() { "i-1000" }, instanceIds);
}
[TestMethod]
public async Task GetInstancesAvailableForTermination_IdleAgentsInMixedAsgs_OnlyReturnInstanceIdFromSameAsgAsync()
{
@@ -170,9 +170,9 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
List<string> instanceIds = await _asgLifecycleService.GetInstancesAvailableForTerminationAsync(e, CancellationToken.None);
// Assert
CollectionAssert.AreEqual(new List<string> () { "i-1000"}, instanceIds);
CollectionAssert.AreEqual(new List<string>() { "i-1000" }, instanceIds);
}
[TestMethod]
public async Task GetInstancesAvailableForTermination_AgentRunningJob_ReturnsNoInstanceIdAsync()
{
@@ -187,7 +187,7 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
// Assert
Assert.AreEqual(0, instanceIds.Count);
}
[TestMethod]
public void DeserializeLifecycleActionEvent()
{
@@ -215,7 +215,7 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
Assert.AreEqual("autoscaling:EC2_INSTANCE_TERMINATING", ev.LifecycleTransition);
Assert.AreEqual("AutoScalingGroup", ev.Origin);
}
[TestMethod]
public void DeserializeTerminationPolicyEvent()
{
@@ -254,14 +254,14 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
Assert.AreEqual(5, ev.CapacityToTerminate[0].Capacity);
Assert.AreEqual("on-demand", ev.CapacityToTerminate[0].InstanceMarketOption);
}
{
Assert.AreEqual(2, ev.Instances.Count);
Assert.AreEqual("us-east-1b", ev.Instances[0].AvailabilityZone);
Assert.AreEqual("i-10001", ev.Instances[0].InstanceId);
Assert.AreEqual("m5d.large", ev.Instances[0].InstanceType);
Assert.AreEqual("on-demand", ev.Instances[0].InstanceMarketOption);
Assert.AreEqual("us-east-1c", ev.Instances[1].AvailabilityZone);
Assert.AreEqual("i-10002", ev.Instances[1].InstanceId);
Assert.AreEqual("m6i.large", ev.Instances[1].InstanceType);
@@ -277,13 +277,15 @@ public class AwsAutoScalingLifecycleServiceTest : TestSetup
Assert.AreEqual(expectedEvent.LifecycleHookName, actual.LifecycleHookName);
Assert.AreEqual(expectedEvent.AutoScalingGroupName, actual.AutoScalingGroupName);
}
private static TerminationPolicyEvent CreateTerminationPolicyEvent(params string[] instanceIds)
{
return new()
{
AutoScalingGroupArn = "test-asg-arn", AutoScalingGroupName = "test-asg-name", Cause = "SCALE_IN",
CapacityToTerminate = new () { new (1) },
AutoScalingGroupArn = "test-asg-arn",
AutoScalingGroupName = "test-asg-name",
Cause = "SCALE_IN",
CapacityToTerminate = new() { new(1) },
Instances = new(instanceIds.Select(x => new TerminationPolicyInstance(x)))
};
}

View File

@@ -6,6 +6,7 @@ using System.Text;
using System.Threading;
using System.Threading.Tasks;
using EpicGames.Core;
using EpicGames.Horde.Agents;
using EpicGames.Horde.Agents.Pools;
using EpicGames.Horde.Jobs.Templates;
using EpicGames.Horde.Streams;

View File

@@ -15,6 +15,7 @@ using Horde.Server.Agents.Pools;
using Horde.Server.Streams;
using EpicGames.Horde.Users;
using EpicGames.Horde.Agents.Pools;
using EpicGames.Horde.Agents;
namespace Horde.Server.Tests.Jobs
{

View File

@@ -149,6 +149,7 @@ namespace Horde.Server.Agents
agent.Id,
agent.Id.ToString(),
agent.Enabled,
agent.Status,
rate,
agent.SessionId,
agent.Ephemeral,

View File

@@ -8,6 +8,7 @@ using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using EpicGames.Horde.Agents;
using EpicGames.Horde.Agents.Pools;
using Horde.Server.Agents.Leases;
using Horde.Server.Agents.Pools;

View File

@@ -22,7 +22,6 @@ using Horde.Server.Agents.Pools;
using Horde.Server.Perforce;
using Horde.Server.Server;
using Horde.Server.Streams;
using HordeCommon;
using HordeCommon.Rpc;
using HordeCommon.Rpc.Messages;
using HordeCommon.Rpc.Tasks;
@@ -65,7 +64,7 @@ namespace Horde.Server.Agents
/// Whether to use an incremental workspace
/// </summary>
public bool Incremental { get; set; }
/// <summary>
/// Method to use when syncing/materializing data from Perforce
/// </summary>
@@ -170,7 +169,7 @@ namespace Horde.Server.Agents
// Construct the message
AgentWorkspace result = new AgentWorkspace
{
ConfiguredCluster = Cluster,
ConfiguredCluster = Cluster,
ConfiguredUserName = UserName,
ServerAndPort = server.ServerAndPort,
UserName = credentials?.UserName ?? UserName,
@@ -182,12 +181,12 @@ namespace Horde.Server.Agents
Partitioned = server.SupportsPartitionedWorkspaces,
Method = Method ?? String.Empty
};
if (View != null)
{
result.View.AddRange(View);
}
return result;
}
}
@@ -407,7 +406,7 @@ namespace Horde.Server.Agents
{
HordeCommon.Rpc.Messages.Lease lease = new HordeCommon.Rpc.Messages.Lease();
lease.Id = Id.ToString();
lease.Payload = Google.Protobuf.WellKnownTypes.Any.Parser.ParseFrom(Payload);
lease.Payload = Google.Protobuf.WellKnownTypes.Any.Parser.ParseFrom(Payload);
lease.State = (RpcLeaseState)State;
return lease;
}
@@ -442,7 +441,7 @@ namespace Horde.Server.Agents
/// Pools requested by the agent to join when registering with server
/// </summary>
public const string RequestedPools = "RequestedPools";
/// <summary>
/// Number of logical cores
/// </summary>
@@ -452,12 +451,12 @@ namespace Horde.Server.Agents
/// Amount of RAM, in GB
/// </summary>
public const string Ram = "RAM";
/// <summary>
/// AWS: Instance ID
/// </summary>
public const string AwsInstanceId = "aws-instance-id";
/// <summary>
/// AWS: Instance type
/// </summary>
@@ -488,7 +487,7 @@ namespace Horde.Server.Agents
/// Current status of this agent
/// </summary>
public AgentStatus Status { get; }
/// <summary>
/// Time at which last status change took place.
/// </summary>
@@ -498,7 +497,7 @@ namespace Horde.Server.Agents
/// Whether the agent is enabled
/// </summary>
public bool Enabled { get; }
/// <summary>
/// Whether the agent is ephemeral
/// </summary>
@@ -624,22 +623,22 @@ namespace Horde.Server.Agents
/// Default tool ID for agent software (multi-platform, shipped without a .NET runtime)
/// This is being deprecated in favor of the platform-specific and self-contained versions of the agent below
/// </summary>
public static ToolId AgentToolId { get; } = new ("horde-agent");
public static ToolId AgentToolId { get; } = new("horde-agent");
/// <summary>
/// Tool ID for Windows-specific and self-contained agent software
/// </summary>
public static ToolId AgentWinX64ToolId { get; } = new ("horde-agent-win-x64");
public static ToolId AgentWinX64ToolId { get; } = new("horde-agent-win-x64");
/// <summary>
/// Tool ID for Linux-specific and self-contained agent software
/// </summary>
public static ToolId AgentLinuxX64ToolId { get; } = new ("horde-agent-linux-x64");
public static ToolId AgentLinuxX64ToolId { get; } = new("horde-agent-linux-x64");
/// <summary>
/// Tool ID for Mac-specific and self-contained agent software
/// </summary>
public static ToolId AgentMacX64ToolId { get; } = new ("horde-agent-osx-x64");
public static ToolId AgentMacX64ToolId { get; } = new("horde-agent-osx-x64");
/// <summary>
/// Gets the tool ID for the software the given agent should be running
@@ -665,7 +664,7 @@ namespace Horde.Server.Agents
{
// Skip support for condition-based software configs below by returning early when self-contained
// Getting this wrong can lead to a self-contained agent getting non-self-contained updates and vice versa.
return agent.GetOsFamily() switch
return agent.GetOsFamily() switch
{
RuntimePlatform.Type.Windows => AgentWinX64ToolId,
RuntimePlatform.Type.Linux => AgentLinuxX64ToolId,
@@ -673,7 +672,7 @@ namespace Horde.Server.Agents
_ => throw new ArgumentOutOfRangeException("Unknown platform " + agent.GetOsFamily())
};
}
foreach (AgentSoftwareConfig softwareConfig in globalConfig.Software)
{
if (softwareConfig.Condition != null && agent.SatisfiesCondition(softwareConfig.Condition))
@@ -721,7 +720,7 @@ namespace Horde.Server.Agents
yield return poolId;
}
}
/// <summary>
/// Tests whether an agent has reported as being a self-contained .NET package
/// </summary>
@@ -732,7 +731,7 @@ namespace Horde.Server.Agents
List<string> values = agent.GetPropertyValues(KnownPropertyNames.SelfContained).ToList();
return values.Count > 0 && values[0].Equals("true", StringComparison.OrdinalIgnoreCase);
}
/// <summary>
/// Get operating system family of agent
/// </summary>
@@ -912,7 +911,7 @@ namespace Horde.Server.Agents
public static AgentWorkspaceInfo? GetAutoSdkWorkspace(this IAgent agent, PerforceCluster cluster, IEnumerable<IPool> pools)
{
AutoSdkConfig? autoSdkConfig = null;
foreach(IPool pool in pools)
foreach (IPool pool in pools)
{
autoSdkConfig = AutoSdkConfig.Merge(autoSdkConfig, pool.AutoSdkConfig);
}
@@ -961,7 +960,7 @@ namespace Horde.Server.Agents
bool partitioned;
AgentWorkspace? existingWorkspace = workspaceMessages.FirstOrDefault(x => x.ConfiguredCluster == workspace.Cluster);
if(existingWorkspace != null)
if (existingWorkspace != null)
{
baseServerAndPort = existingWorkspace.BaseServerAndPort;
serverAndPort = existingWorkspace.ServerAndPort;
@@ -1002,7 +1001,7 @@ namespace Horde.Server.Agents
// Construct the message
AgentWorkspace result = new AgentWorkspace
{
ConfiguredCluster = workspace.Cluster,
ConfiguredCluster = workspace.Cluster,
ConfiguredUserName = workspace.UserName,
Cluster = cluster?.Name,
BaseServerAndPort = baseServerAndPort,
@@ -1021,7 +1020,7 @@ namespace Horde.Server.Agents
{
result.View.AddRange(workspace.View);
}
workspaceMessages.Add(result);
return true;
}

View File

@@ -2,14 +2,13 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Horde.Server.Auditing;
using HordeCommon;
using EpicGames.Horde.Agents.Leases;
using EpicGames.Horde.Agents;
using EpicGames.Horde.Agents.Leases;
using EpicGames.Horde.Agents.Pools;
using EpicGames.Horde.Agents.Sessions;
using System.Threading;
using Horde.Server.Auditing;
namespace Horde.Server.Agents
{

View File

@@ -323,7 +323,7 @@ namespace Horde.Server.Server
GetCapabilities(request.Capabilities, out List<string> properties, out Dictionary<string, int> resources);
// Create a new session
agent = await _agentService.CreateSessionAsync(agent, request.Status, properties, resources, request.Version, context.CancellationToken);
agent = await _agentService.CreateSessionAsync(agent, (AgentStatus)request.Status, properties, resources, request.Version, context.CancellationToken);
if (agent == null)
{
throw new StructuredRpcException(StatusCode.NotFound, "Agent {AgentId} not found", agentId);
@@ -396,7 +396,7 @@ namespace Horde.Server.Server
// Update the session
try
{
agent = await _agentService.UpdateSessionWithWaitAsync(agent, sessionId, request.Status, properties, resources, request.Leases, cancellationSource.Token);
agent = await _agentService.UpdateSessionWithWaitAsync(agent, sessionId, (AgentStatus)request.Status, properties, resources, request.Leases, cancellationSource.Token);
}
catch (OperationCanceledException)
{
@@ -421,7 +421,7 @@ namespace Horde.Server.Server
UpdateSessionResponse response = new UpdateSessionResponse();
response.Leases.Add(agent.Leases.Select(x => x.ToRpcMessage()));
response.ExpiryTime = (agent.SessionExpiresAt == null) ? new Timestamp() : Timestamp.FromDateTime(agent.SessionExpiresAt.Value);
response.Status = agent.Status;
response.Status = (RpcAgentStatus)agent.Status;
await writer.WriteAsync(response);
}

View File

@@ -8,6 +8,42 @@ using EpicGames.Horde.Logs;
namespace EpicGames.Horde.Agents
{
/// <summary>
/// Status of an agent. Must match RpcAgentStatus.
/// </summary>
public enum AgentStatus
{
/// <summary>
/// Unspecified state.
/// </summary>
Unspecified = 0,
/// <summary>
/// Agent is running normally.
/// </summary>
Ok = 1,
/// <summary>
/// Agent is currently shutting down, and should not be assigned new leases.
/// </summary>
Stopping = 2,
/// <summary>
/// Agent is in an unhealthy state and should not be assigned new leases.
/// </summary>
Unhealthy = 3,
/// <summary>
/// Agent is currently stopped.
/// </summary>
Stopped = 4,
/// <summary>
/// Agent is busy performing other work (eg. serving an interactive user)
/// </summary>
Busy = 5,
}
/// <summary>
/// Parameters to update an agent
/// </summary>
@@ -67,6 +103,7 @@ namespace EpicGames.Horde.Agents
/// <param name="Id"> The agent's unique ID </param>
/// <param name="Name"> Friendly name of the agent </param>
/// <param name="Enabled"> Whether the agent is currently enabled </param>
/// <param name="Status">Status of the agent</param>
/// <param name="Rate"> Cost estimate per-hour for this agent </param>
/// <param name="SessionId"> The current session id </param>
/// <param name="Ephemeral"> Whether the agent is ephemeral </param>
@@ -90,5 +127,5 @@ namespace EpicGames.Horde.Agents
/// <param name="Leases"> Array of active leases. </param>
/// <param name="Workspaces">Current workspaces synced on the agent</param>
/// <param name="Comment"> Comment for this agent </param>
public record GetAgentResponse(AgentId Id, string Name, bool Enabled, double? Rate, SessionId? SessionId, bool Ephemeral, bool Online, bool Deleted, bool PendingConform, bool PendingFullConform, bool PendingRestart, bool PendingShutdown, string LastShutdownReason, DateTime LastConformTime, int? ConformAttemptCount, DateTime? NextConformTime, string? Version, List<string> Properties, Dictionary<string, int> Resources, DateTime? UpdateTime, DateTime? LastStatusChange, List<string>? Pools, object? Capabilities, List<GetAgentLeaseResponse> Leases, List<GetAgentWorkspaceResponse> Workspaces, string? Comment);
public record GetAgentResponse(AgentId Id, string Name, bool Enabled, AgentStatus Status, double? Rate, SessionId? SessionId, bool Ephemeral, bool Online, bool Deleted, bool PendingConform, bool PendingFullConform, bool PendingRestart, bool PendingShutdown, string LastShutdownReason, DateTime LastConformTime, int? ConformAttemptCount, DateTime? NextConformTime, string? Version, List<string> Properties, Dictionary<string, int> Resources, DateTime? UpdateTime, DateTime? LastStatusChange, List<string>? Pools, object? Capabilities, List<GetAgentLeaseResponse> Leases, List<GetAgentWorkspaceResponse> Workspaces, string? Comment);
}