// Copyright Epic Games, Inc. All Rights Reserved. using Google.Protobuf.WellKnownTypes; using HordeServer.Api; using HordeCommon; using HordeCommon.Rpc.Tasks; using HordeServer.Services; using HordeServer.Utilities; using MongoDB.Bson; using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Attributes; using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Security.Cryptography; using Serilog; using PoolId = HordeServer.Utilities.StringId; using StreamId = HordeServer.Utilities.StringId; using AgentSoftwareVersion = HordeServer.Utilities.StringId; using AgentSoftwareChannelName = HordeServer.Utilities.StringId; using System.Diagnostics; using System.Threading.Tasks; namespace HordeServer.Models { /// /// Information about a workspace synced to an agent /// public class AgentWorkspace { /// /// Name of the Perforce cluster to use /// public string? Cluster { get; set; } /// /// User to log into Perforce with (eg. buildmachine) /// public string? UserName { get; set; } /// /// Identifier to distinguish this workspace from other workspaces /// public string Identifier { get; set; } /// /// The stream to sync /// public string Stream { get; set; } /// /// Custom view for the workspace /// public List? View { get; set; } /// /// Whether to use an incremental workspace /// public bool bIncremental { get; set; } /// /// Constructor /// /// Name of the Perforce cluster /// User to log into Perforce with (eg. buildmachine) /// Identifier to distinguish this workspace from other workspaces /// The stream to sync /// Custom view for the workspace /// Whether to use an incremental workspace public AgentWorkspace(string? Cluster, string? UserName, string Identifier, string Stream, List? View, bool bIncremental) { if (!String.IsNullOrEmpty(Cluster)) { this.Cluster = Cluster; } if (!String.IsNullOrEmpty(UserName)) { this.UserName = UserName; } this.Identifier = Identifier; this.Stream = Stream; this.View = View; this.bIncremental = bIncremental; } /// /// Constructor /// /// RPC message to construct from public AgentWorkspace(HordeCommon.Rpc.Messages.AgentWorkspace Workspace) : this(Workspace.ConfiguredCluster, Workspace.ConfiguredUserName, Workspace.Identifier, Workspace.Stream, (Workspace.View.Count > 0) ? Workspace.View.ToList() : null, Workspace.Incremental) { } /// /// Gets a digest of the settings for this workspace /// /// Digest for the workspace settings public string GetDigest() { #pragma warning disable CA5351 // Do Not Use Broken Cryptographic Algorithms using (MD5 Hasher = MD5.Create()) { byte[] Data = BsonExtensionMethods.ToBson(this); return BitConverter.ToString(Hasher.ComputeHash(Data)).Replace("-", "", StringComparison.Ordinal); } #pragma warning restore CA5351 } /// public override bool Equals(object? Obj) { AgentWorkspace? Other = Obj as AgentWorkspace; if (Other == null) { return false; } if (Cluster != Other.Cluster || UserName != Other.UserName || Identifier != Other.Identifier || Stream != Other.Stream || bIncremental != Other.bIncremental) { return false; } if (!Enumerable.SequenceEqual(View ?? new List(), Other.View ?? new List())) { return false; } return true; } /// public override int GetHashCode() { return HashCode.Combine(Cluster, UserName, Identifier, Stream, bIncremental); // Ignore 'View' for now } /// /// Checks if two workspace sets are equivalent, ignoring order /// /// First list of workspaces /// Second list of workspaces /// True if the sets are equivalent public static bool SetEquals(IReadOnlyList WorkspacesA, IReadOnlyList WorkspacesB) { HashSet WorkspacesSetA = new HashSet(WorkspacesA); return WorkspacesSetA.SetEquals(WorkspacesB); } /// /// Converts this workspace to an RPC message /// /// The Perforce server /// Credentials for the server /// The RPC message public HordeCommon.Rpc.Messages.AgentWorkspace ToRpcMessage(IPerforceServer Server, PerforceCredentials? Credentials) { // Construct the message HordeCommon.Rpc.Messages.AgentWorkspace Result = new HordeCommon.Rpc.Messages.AgentWorkspace(); Result.ConfiguredCluster = Cluster; Result.ConfiguredUserName = UserName; Result.ServerAndPort = Server.ServerAndPort; Result.UserName = Credentials?.UserName ?? UserName; Result.Password = Credentials?.Password; Result.Identifier = Identifier; Result.Stream = Stream; if (View != null) { Result.View.AddRange(View); } Result.Incremental = bIncremental; return Result; } } /// /// Information about a device allocated for a lease /// public class AgentLeaseDevice { /// /// Index of the device in the agent's device list /// [BsonRequired] public int Index { get; set; } /// /// Handle of the device in the lease, ie. the logical name in the context of the work being done, not the physical device name. /// [BsonRequired] public string? Handle { get; set; } /// /// Resources claimed by the lease. If null, the whole device is claimed, and nothing else can be allocated to it. /// [BsonIgnoreIfNull] public Dictionary? Resources { get; set; } /// /// Private constructor for serialization /// [BsonConstructor] private AgentLeaseDevice() { Handle = null!; } /// /// Constructor /// /// /// /// public AgentLeaseDevice(int Index, string? Handle, Dictionary? Resources) { this.Index = Index; this.Handle = Handle; this.Resources = Resources; } } /// /// Document describing an active lease /// public class AgentLease { /// /// Name of this lease /// [BsonRequired] public ObjectId Id { get; set; } /// /// Name of this lease /// public string Name { get; set; } /// /// The current state of the lease /// public LeaseState State { get; set; } /// /// The stream for this lease /// public StreamId? StreamId { get; set; } /// /// The pool for this lease /// public PoolId? PoolId { get; set; } /// /// Optional log for this lease /// public ObjectId? LogId { get; set; } /// /// Time at which the lease started /// [BsonRequired] public DateTime StartTime { get; set; } /// /// Time at which the lease should be terminated /// public DateTime? ExpiryTime { get; set; } /// /// Flag indicating whether this lease has been accepted by the agent /// public bool Active { get; set; } /// /// Requirements for this lease /// public AgentRequirements Requirements { get; set; } /// /// For leases in the pending state, encodes an "any" protobuf containing the payload for the agent to execute the lease. /// public byte[]? Payload { get; set; } /// /// List of devices allocated to this lease /// [BsonIgnoreIfNull] public List? Devices { get; set; } /// /// Private constructor /// [BsonConstructor] private AgentLease() { Name = String.Empty; Requirements = null!; Devices = null!; } /// /// Constructor /// /// Identifier for the lease /// Name of this lease /// /// /// Unique id for the log /// State for the lease /// Encoded "any" protobuf describing the contents of the payload /// Requirements for this lease /// Device mapping for this lease public AgentLease(ObjectId Id, string Name, StreamId? StreamId, PoolId? PoolId, ObjectId? LogId, LeaseState State, byte[]? Payload, AgentRequirements Requirements, List? Devices) { this.Id = Id; this.Name = Name; this.StreamId = StreamId; this.PoolId = PoolId; this.LogId = LogId; this.State = State; this.Payload = Payload; this.Requirements = Requirements; this.Devices = Devices; StartTime = DateTime.UtcNow; } /// /// Determines if this is a conform lease /// /// True if this is a conform lease public bool IsConformLease() { if (Payload != null) { Any BasePayload = Any.Parser.ParseFrom(Payload); if (BasePayload.Is(ConformTask.Descriptor)) { return true; } } return false; } /// /// Gets user-readable payload information /// /// The payload data /// Dictionary of key/value pairs for the payload public static Dictionary? GetPayloadDetails(ReadOnlyMemory? Payload) { Dictionary? Details = null; if (Payload != null) { Any BasePayload = Any.Parser.ParseFrom(Payload.Value.ToArray()); Details = new Dictionary(); ConformTask ConformTask; if(BasePayload.TryUnpack(out ConformTask)) { Details["Type"] = "Conform"; Details["LogId"] = ConformTask.LogId; } ExecuteJobTask JobTask; if (BasePayload.TryUnpack(out JobTask)) { Details["Type"] = "Job"; Details["JobId"] = JobTask.JobId; Details["BatchId"] = JobTask.BatchId; Details["LogId"] = JobTask.LogId; } UpgradeTask UpgradeTask; if (BasePayload.TryUnpack(out UpgradeTask)) { Details["Type"] = "Upgrade"; Details["SoftwareId"] = UpgradeTask.SoftwareId; Details["LogId"] = UpgradeTask.LogId; } } return Details; } /// /// Converts this lease to an RPC message /// /// RPC message public HordeCommon.Rpc.Messages.Lease ToRpcMessage() { HordeCommon.Rpc.Messages.Lease Lease = new HordeCommon.Rpc.Messages.Lease(); Lease.Id = Id.ToString(); Lease.Payload = Google.Protobuf.WellKnownTypes.Any.Parser.ParseFrom(Payload); Lease.State = State; return Lease; } } /// /// Mirrors an Agent document in the database /// public interface IAgent { /// /// Randomly generated unique id for this agent. /// public AgentId Id { get; } /// /// The current session id, if it's online /// public ObjectId? SessionId { get; } /// /// Time at which the current session expires. /// public DateTime? SessionExpiresAt { get; } /// /// Current status of this agent /// public AgentStatus Status { get; } /// /// Whether the agent is enabled /// public bool Enabled { get; } /// /// Arbitrary comment for the agent (useful for disable reasons etc) /// public string? Comment { get; } /// /// Whether the agent is ephemeral /// public bool Ephemeral { get; } /// /// Whether the agent should be included on the dashboard. This is set to true for ephemeral agents once they are no longer online, or agents that are explicitly deleted. /// public bool Deleted { get; } /// /// Version of the software running on this agent /// public string? Version { get; } /// /// Channel for the software running on this agent. Uses if not specified /// public AgentSoftwareChannelName? Channel { get; } /// /// Last upgrade that was attempted /// public string? LastUpgradeVersion { get; } /// /// Time that which the last upgrade was attempted /// public DateTime? LastUpgradeTime { get; } /// /// List of manually assigned pools for agent /// public IReadOnlyList ExplicitPools { get; } /// /// Whether a conform is requested /// public bool RequestConform { get; } /// /// Whether a machine restart is requested /// public bool RequestRestart { get; } /// /// Whether the machine should be shutdown /// public bool RequestShutdown { get; } /// /// List of workspaces currently synced to this machine /// public IReadOnlyList Workspaces { get; } /// /// Time at which the last conform job ran /// public DateTime LastConformTime { get; } /// /// Number of times a conform job has failed /// public int? ConformAttemptCount { get; } /// /// Capabilities of this agent /// public AgentCapabilities Capabilities { get; } /// /// Array of active leases. /// public IReadOnlyList Leases { get; } /// /// ACL for modifying this agent /// public Acl? Acl { get; } /// /// Last time that the agent was modified /// public DateTime UpdateTime { get; } /// /// Update counter for this document. Any updates should compare-and-swap based on the value of this counter, or increment it in the case of server-side updates. /// public uint UpdateIndex { get; } } /// /// Extension methods for IAgent /// public static class AgentExtensions { /// /// Determines whether this agent is online /// /// public static bool IsSessionValid(this IAgent Agent, DateTime UtcNow) { return Agent.SessionId.HasValue && Agent.SessionExpiresAt.HasValue && UtcNow < Agent.SessionExpiresAt.Value; } /// /// Gets a list of pools for the given agent. Includes all automatically assigned pools based on agent capabilities. /// /// The agent instance /// List of all available pools /// List of pools public static IEnumerable GetPools(this IAgent Agent, IEnumerable Pools) { return Pools.Where(x => Agent.InPool(x)); } /// /// Checks whether an agent is in the given pool /// /// The agent to check /// The pool to test against /// True if the agent is in the pool public static bool InPool(this IAgent Agent, IPool Pool) { if (Agent.ExplicitPools.Contains(Pool.Id)) { return true; } else if (Pool.Requirements != null && AgentExtensions.TryCreateLease(Agent.Capabilities, Pool.Requirements, Enumerable.Empty(), out _)) { return true; } return false; } /// /// Determine whether it's possible to add a lease for the given resources /// /// The agent to create a lease for /// The pool required /// Requirements for this lease /// On success, recieves the list of leased devices /// True if the new lease can be granted public static bool TryCreateLease(this IAgent Agent, IPool Pool, AgentRequirements? Requirements, out List? LeasedDevices) { if (!Agent.Enabled || Agent.Status != AgentStatus.Ok || !Agent.InPool(Pool)) { LeasedDevices = null!; return false; } return TryCreateLease(Agent.Capabilities, Requirements, Agent.Leases, out LeasedDevices); } /// /// Attempts to match the requirements for a particular agent, and return the list of leased devices that fulfils the requirements /// /// Capabilities of the gent /// Requirements for the lease /// List of current leases /// Receives the list of device leases /// True if successful public static bool TryCreateLease(AgentCapabilities Capabilities, AgentRequirements? Requirements, IEnumerable Leases, out List? LeasedDevices) { List? Result = null; if (Requirements != null) { if (!Requirements.Shared && Leases.Any()) { LeasedDevices = null; return false; } if (Requirements.Properties != null) { foreach (string Property in Requirements.Properties) { if (Capabilities.Properties == null || !Capabilities.Properties.Contains(Property)) { LeasedDevices = null!; return false; } } } if (Requirements.Devices != null) { if (!TryCreateDeviceLeases(Capabilities.Devices, Requirements.Devices, 0, 0, Leases.SelectMany(x => x.Devices), out Result)) { LeasedDevices = null!; return false; } } } LeasedDevices = Result; return true; } /// /// Try to match up the required devices for a lease to the available devices on an agent. This is done through a recursive search with backtracking for completeness, /// due to not knowing which devices in the lease will correspond to each device in the agent, but it should be quick in practice due to agents having a small number /// of homogenous devices. /// /// Device capabilities for the agent /// Device requirements for the lease /// Current index of the device in the lease requirements /// Bitmask for devices that have currently been allocated. This is limited to 32, which is unlikely to be a problem in practice. /// The current set of leased devices /// On success, receives a list of the leased devices, mapping each requested device to a device in the agent device array. /// True if the leases were allocated private static bool TryCreateDeviceLeases(List CapableDevices, List RequiredDevices, int RequiredDeviceIdx, uint LeasedDeviceMask, IEnumerable CurrentLeasedDevices, [MaybeNullWhen(false)] out List LeasedDevices) { // If we've assigned all the required devices now, we can create the list of assigned devices if (RequiredDeviceIdx == RequiredDevices.Count) { LeasedDevices = new List(RequiredDevices.Count); foreach (DeviceRequirements Device in RequiredDevices) { LeasedDevices.Add(new AgentLeaseDevice(-1, Device.Handle, Device.Resources)); } return true; } // Otherwise try to assign the next device DeviceRequirements RequiredDevice = RequiredDevices[RequiredDeviceIdx]; for (int DeviceIdx = 0; DeviceIdx < CapableDevices.Count; DeviceIdx++) { uint DeviceFlag = 1U << DeviceIdx; if ((LeasedDeviceMask & DeviceFlag) == 0 && MatchDevice(CapableDevices[DeviceIdx], RequiredDevice, CurrentLeasedDevices.Where(x => x.Index == DeviceIdx))) { List? Result; if (TryCreateDeviceLeases(CapableDevices, RequiredDevices, RequiredDeviceIdx + 1, LeasedDeviceMask | DeviceFlag, CurrentLeasedDevices, out Result)) { LeasedDevices = Result; LeasedDevices[RequiredDeviceIdx].Index = DeviceIdx; return true; } } } // Otherwise failed LeasedDevices = null!; return false; } /// /// Determines if an agent device has the given requirements /// /// The device capabilities /// Requirements for the lease /// Current leases for the device /// True if the device can satisfy the given requirements private static bool MatchDevice(DeviceCapabilities Capabilities, DeviceRequirements Requirements, IEnumerable Leases) { // Check the device has all the required properties if (Requirements.Properties != null) { foreach (string Property in Requirements.Properties) { if (Capabilities.Properties == null || !Capabilities.Properties.Contains(Property)) { return false; } } } // Check the device has all the required resources if (Requirements.Resources == null) { // Requires exclusive access to the device if (Leases.Any()) { return false; } } else { // Requires shared access to the device. Check the device can be shared. if (Capabilities.Resources == null) { return false; } // Check there are enough available of each resource type foreach (KeyValuePair Resource in Requirements.Resources) { // Make sure the device has enough of the named resource to start with int RemainingCount; if (!Capabilities.Resources.TryGetValue(Resource.Key, out RemainingCount) || RemainingCount < Resource.Value) { return false; } // Check each existing lease of this device foreach (AgentLeaseDevice Lease in Leases) { // If the lease has an exclusive reservation, we can't use it if (Lease.Resources == null) { return false; } // Update the remaining count for this resource int UsedCount; if (Lease.Resources.TryGetValue(Resource.Key, out UsedCount)) { RemainingCount -= UsedCount; if (RemainingCount < Resource.Value) { return false; } } } } } return true; } /// /// Gets all the autosdk workspaces required for an agent /// /// /// /// /// public static HashSet GetAutoSdkWorkspaces(this IAgent Agent, Globals Globals, List Workspaces) { HashSet AutoSdkWorkspaces = new HashSet(); foreach (string? ClusterName in Workspaces.Select(x => x.Cluster).Distinct()) { PerforceCluster? Cluster = Globals.FindPerforceCluster(ClusterName); if (Cluster != null) { AgentWorkspace? AutoSdkWorkspace = GetAutoSdkWorkspace(Agent, Cluster); if (AutoSdkWorkspace != null) { AutoSdkWorkspaces.Add(AutoSdkWorkspace); } } } return AutoSdkWorkspaces; } /// /// Get the AutoSDK workspace required for an agent /// /// /// The perforce cluster to get a workspace for /// public static AgentWorkspace? GetAutoSdkWorkspace(this IAgent Agent, PerforceCluster Cluster) { if (Agent.Capabilities.Devices.Count > 0) { DeviceCapabilities PrimaryDevice = Agent.Capabilities.Devices[0]; if (PrimaryDevice.Properties != null) { foreach (AutoSdkWorkspace AutoSdk in Cluster.AutoSdk) { if (AutoSdk.Stream != null && AutoSdk.Properties.All(x => PrimaryDevice.Properties.Contains(x))) { return new AgentWorkspace(Cluster.Name, AutoSdk.UserName, AutoSdk.Name ?? "AutoSDK", AutoSdk.Stream!, null, true); } } } } return null; } /// /// Converts this workspace to an RPC message /// /// The agent to get a workspace for /// The workspace definition /// The global state /// The Perforce load balancer /// List of messages /// The RPC message public static async Task TryAddWorkspaceMessage(this IAgent Agent, AgentWorkspace Workspace, PerforceCluster Cluster, PerforceLoadBalancer LoadBalancer, IList WorkspaceMessages) { // Find a matching server, trying to use a previously selected one if possible string? BaseServerAndPort; string? ServerAndPort; HordeCommon.Rpc.Messages.AgentWorkspace? ExistingWorkspace = WorkspaceMessages.FirstOrDefault(x => x.ConfiguredCluster == Workspace.Cluster); if(ExistingWorkspace != null) { BaseServerAndPort = ExistingWorkspace.BaseServerAndPort; ServerAndPort = ExistingWorkspace.ServerAndPort; } else { if (Cluster == null) { return false; } IPerforceServer? Server = await LoadBalancer.SelectServerAsync(Cluster, Agent); if (Server == null) { return false; } BaseServerAndPort = Server.BaseServerAndPort; ServerAndPort = Server.ServerAndPort; } // Find the matching credentials for the desired user PerforceCredentials? Credentials = null; if (Cluster != null) { if (Workspace.UserName == null) { Credentials = Cluster.Credentials.FirstOrDefault(); } else { Credentials = Cluster.Credentials.FirstOrDefault(x => String.Equals(x.UserName, Workspace.UserName, StringComparison.OrdinalIgnoreCase)); } } // Construct the message HordeCommon.Rpc.Messages.AgentWorkspace Result = new HordeCommon.Rpc.Messages.AgentWorkspace(); Result.ConfiguredCluster = Workspace.Cluster; Result.ConfiguredUserName = Workspace.UserName; Result.Cluster = Cluster?.Name; Result.BaseServerAndPort = BaseServerAndPort; Result.ServerAndPort = ServerAndPort; Result.UserName = Credentials?.UserName ?? Workspace.UserName; Result.Password = Credentials?.Password; Result.Identifier = Workspace.Identifier; Result.Stream = Workspace.Stream; if (Workspace.View != null) { Result.View.AddRange(Workspace.View); } Result.Incremental = Workspace.bIncremental; WorkspaceMessages.Add(Result); return true; } } }