// Copyright Epic Games, Inc. All Rights Reserved. using EpicGames.Core; using Google.Protobuf.WellKnownTypes; using HordeServer.Api; using HordeCommon; using HordeServer.Utilities; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using MongoDB.Driver; using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using PoolId = HordeServer.Utilities.StringId; using ProjectId = HordeServer.Utilities.StringId; using StreamId = HordeServer.Utilities.StringId; using TemplateRefId = HordeServer.Utilities.StringId; using System.Runtime.CompilerServices; namespace HordeServer.Models { /// /// Exception thrown when stream validation fails /// public class InvalidStreamException : Exception { /// public InvalidStreamException() { } /// public InvalidStreamException(string Message) : base(Message) { } /// public InvalidStreamException(string Message, Exception InnerEx) : base(Message, InnerEx) { } } /// /// Mapping from a BuildGraph agent type to a set of machines on the farm /// public class AgentType { /// /// Name of the pool of agents to use /// [BsonRequired] public PoolId Pool { get; set; } /// /// Name of the workspace to execute on /// public string? Workspace { get; set; } /// /// Path to the temporary storage dir /// public string? TempStorageDir { get; set; } /// /// Environment variables to be set when executing the job /// public Dictionary? Environment { get; set; } /// /// Default constructor /// [BsonConstructor] private AgentType() { } /// /// Constructor /// /// The pool for this agent /// Name of the workspace to use /// Path to the temp storage directory public AgentType(PoolId Pool, string Workspace, string? TempStorageDir) { this.Pool = Pool; this.Workspace = Workspace; this.TempStorageDir = TempStorageDir; } /// /// Constructor /// /// The object to construct from public AgentType(CreateAgentTypeRequest Request) { Pool = new PoolId(Request.Pool); Workspace = Request.Workspace; TempStorageDir = Request.TempStorageDir; Environment = Request.Environment; } /// /// Constructs an AgentType object from an optional request /// /// The request object /// New agent type object [return: NotNullIfNotNull("Request")] public static AgentType? FromRequest(CreateAgentTypeRequest? Request) { return (Request != null) ? new AgentType(Request) : null; } /// /// Creates an API response object from this stream /// /// The response object public Api.GetAgentTypeResponse ToApiResponse() { return new Api.GetAgentTypeResponse(Pool.ToString(), Workspace, TempStorageDir, Environment); } /// /// Creates an API response object from this stream /// /// The response object public HordeCommon.Rpc.GetAgentTypeResponse ToRpcResponse() { HordeCommon.Rpc.GetAgentTypeResponse Response = new HordeCommon.Rpc.GetAgentTypeResponse(); if (TempStorageDir != null) { Response.TempStorageDir = TempStorageDir; } if (Environment != null) { Response.Environment.Add(Environment); } return Response; } } /// /// Information about a workspace type /// public class WorkspaceType { /// /// Name of the Perforce cluster to use /// [BsonIgnoreIfNull] public string? Cluster { get; set; } /// /// The Perforce server and port /// [BsonIgnoreIfNull] public string? ServerAndPort { get; set; } /// /// The Perforce username for syncing this workspace /// [BsonIgnoreIfNull] public string? UserName { get; set; } /// /// The Perforce password for syncing this workspace /// [BsonIgnoreIfNull] public string? Password { get; set; } /// /// Identifier to distinguish this workspace from other workspaces. Defaults to the workspace type name. /// [BsonIgnoreIfNull] public string? Identifier { get; set; } /// /// Override for the stream to sync /// [BsonIgnoreIfNull] public string? Stream { get; set; } /// /// Custom view for the workspace /// [BsonIgnoreIfNull] public List? View { get; set; } /// /// Whether to use an incrementally synced workspace /// public bool Incremental { get; set; } /// /// Default constructor /// public WorkspaceType() { } /// /// Constructor /// /// The object to construct from public WorkspaceType(CreateWorkspaceTypeRequest Request) { Cluster = Request.Cluster; ServerAndPort = Request.ServerAndPort; UserName = Request.UserName; Password = Request.Password; Identifier = Request.Identifier; Stream = Request.Stream; View = Request.View; Incremental = Request.Incremental; } /// /// Constructs an AgentType object from an optional request /// /// The request object /// New agent type object [return: NotNullIfNotNull("Request")] public static WorkspaceType? FromRequest(CreateWorkspaceTypeRequest? Request) { return (Request != null) ? new WorkspaceType(Request) : null; } /// /// Creates an API response object from this stream /// /// The response object public Api.GetWorkspaceTypeResponse ToApiResponse() { return new Api.GetWorkspaceTypeResponse(Cluster, ServerAndPort, UserName, Identifier, Stream, View, Incremental); } } /// /// Allows triggering another downstream job on succesful completion of a step or aggregate /// public class ChainedJobTemplate { /// /// Name of the target that needs to complete successfully /// public string Trigger { get; set; } = String.Empty; /// /// The new template to trigger /// public TemplateRefId TemplateRefId { get; set; } /// /// Default constructor for serialization /// private ChainedJobTemplate() { } /// /// Constructor /// /// Name of the target that needs to complete /// The new template to trigger public ChainedJobTemplate(string Trigger, TemplateRefId TemplateRefId) { this.Trigger = Trigger; this.TemplateRefId = TemplateRefId; } /// /// Constructor /// /// Request to construct from public ChainedJobTemplate(CreateChainedJobTemplateRequest Request) : this(Request.Trigger, new TemplateRefId(Request.TemplateId)) { } } /// /// Reference to a template /// public class TemplateRef { /// /// The template name (duplicated from the template object) /// public string Name { get; set; } /// /// Hash of the template definition /// public ContentHash Hash { get; set; } /// /// Whether to show badges in UGS for this schedule /// public bool ShowUgsBadges { get; set; } /// /// Whether to show desktop alerts for build health issues created from jobs this type /// public bool ShowUgsAlerts { get; set; } /// /// Notification channel for this template. Overrides the stream channel if set. /// public string? NotificationChannel { get; set; } /// /// Notification channel filter for this template. Errors|Warnings|Success /// public string? NotificationChannelFilter { get; set; } /// /// Channel for triage notification messages /// public string? TriageChannel { get; set; } /// /// List of schedules for this template /// [BsonIgnoreIfNull] public Schedule? Schedule { get; set; } /// /// List of downstream templates to trigger at the same change /// [BsonIgnoreIfNull] public List? ChainedJobs { get; set; } /// /// Custom permissions for this template /// public Acl? Acl { get; set; } /// /// Private constructor for serialization /// private TemplateRef() { this.Name = null!; this.Hash = null!; } /// /// Constructor /// /// The template being referenced /// Whether to show badges in UGS for this job /// Whether to show alerts in UGS for this job /// Notification channel for this template /// Notification channel filter for this template /// /// Schedule for this template /// List of downstream templates to trigger /// ACL for this template public TemplateRef(ITemplate Template, bool ShowUgsBadges = false, bool ShowUgsAlerts = false, string? NotificationChannel = null, string? NotificationChannelFilter = null, string? TriageChannel = null, Schedule? Schedule = null, List? Triggers = null, Acl? Acl = null) { this.Name = Template.Name; this.Hash = Template.Id; this.ShowUgsBadges = ShowUgsBadges; this.ShowUgsAlerts = ShowUgsAlerts; this.NotificationChannel = NotificationChannel; this.NotificationChannelFilter = NotificationChannelFilter; this.TriageChannel = TriageChannel; this.Schedule = Schedule; this.ChainedJobs = Triggers; this.Acl = Acl; } } /// /// Query used to identify a base changelist for a preflight /// public class ChangeQuery { /// /// Template to search for /// public TemplateRefId? TemplateRefId { get; set; } /// /// The target to look at the status for /// public string? Target { get; set; } /// /// Whether to match a job that contains warnings /// public List? Outcomes { get; set; } /// /// Convert to a request object /// /// public ChangeQueryRequest ToRequest() { return new ChangeQueryRequest { TemplateId = TemplateRefId?.ToString(), Target = Target, Outcomes = Outcomes }; } } /// /// Definition of a query to execute to find the changelist to run a build at /// public class DefaultPreflight { /// /// The template id to execute /// public TemplateRefId? TemplateRefId { get; set; } /// /// Query specifying a changelist to use /// public ChangeQuery? Change { get; set; } /// /// The job type to query for the change to use /// [Obsolete("Use Change.TemplateRefId instead")] public TemplateRefId? ChangeTemplateRefId { get; set; } /// /// Constructor /// /// /// The job type to query for the change to use public DefaultPreflight(TemplateRefId? TemplateRefId, ChangeQuery? Change) { this.TemplateRefId = TemplateRefId; this.Change = Change; } /// /// Convert to a request object /// /// public DefaultPreflightRequest ToRequest() { #pragma warning disable CS0618 // Type or member is obsolete ChangeQueryRequest? ChangeRequest = null; if (Change != null) { ChangeRequest = Change.ToRequest(); } else if (ChangeTemplateRefId != null) { ChangeRequest = new ChangeQueryRequest { TemplateId = ChangeTemplateRefId?.ToString() }; } return new DefaultPreflightRequest { TemplateId = TemplateRefId?.ToString(), Change = ChangeRequest, ChangeTemplateId = ChangeRequest?.TemplateId }; #pragma warning restore CS0618 // Type or member is obsolete } } /// /// Extension methods for template refs /// static class TemplateRefExtensions { /// /// Adds a new template ref to a list /// /// List of template refs /// The template ref to add public static void AddRef(this Dictionary TemplateRefs, TemplateRef TemplateRef) { TemplateRefs.Add(new TemplateRefId(TemplateRef.Name), TemplateRef); } } /// /// Information about a stream /// public interface IStream { /// /// Name of the stream. /// public StreamId Id { get; } /// /// The project that this stream belongs to /// public ProjectId ProjectId { get; } /// /// The stream name /// public string Name { get; } /// /// Path to the configuration file for this stream /// public string ConfigPath { get; } /// /// The revision of config file used for this stream /// public string ConfigRevision { get; } /// /// Order to display on the dashboard's drop-down list /// public int Order { get; } /// /// Notification channel for all jobs in this stream /// public string? NotificationChannel { get; } /// /// Notification channel filter for all jobs in this stream. Errors|Warnings|Success /// public string? NotificationChannelFilter { get; } /// /// Channel to post issue triage notifications /// public string? TriageChannel { get; } /// /// Default template to use for preflights /// public DefaultPreflight? DefaultPreflight { get; } /// /// List of pages to display in the dashboard /// public IReadOnlyList Tabs { get; } /// /// Dictionary of agent types /// public IReadOnlyDictionary AgentTypes { get; } /// /// Dictionary of workspace types /// public IReadOnlyDictionary WorkspaceTypes { get; } /// /// List of templates available for this stream /// public IReadOnlyDictionary Templates { get; } /// /// Last time that we queried for commits /// public DateTime? LastCommitTime { get; } /// /// Stream is paused for builds until specified time /// public DateTime? PausedUntil { get; } /// /// Comment/reason for why the stream was paused /// public string? PauseComment { get; } /// /// The ACL for this object /// public Acl? Acl { get; } } /// /// Extension methods for streams /// static class StreamExtensions { /// /// Tries to get an agent workspace definition from the given type name /// /// The stream object /// The agent type /// Receives the agent workspace definition /// True if the agent type was valid, and an agent workspace could be created public static bool TryGetAgentWorkspace(this IStream Stream, AgentType AgentType, [NotNullWhen(true)] out AgentWorkspace? Workspace) { // Get the workspace settings if (AgentType.Workspace == null) { // Use the default settings (fast switching workspace, clean Workspace = new AgentWorkspace(null, null, GetDefaultWorkspaceIdentifier(Stream), Stream.Name, null, false); return true; } else { // Try to get the matching workspace type WorkspaceType? WorkspaceType; if (!Stream.WorkspaceTypes.TryGetValue(AgentType.Workspace, out WorkspaceType)) { Workspace = null; return false; } // Get the workspace identifier string Identifier; if (WorkspaceType.Identifier != null && !String.IsNullOrEmpty(WorkspaceType.Identifier)) { Identifier = WorkspaceType.Identifier; } else if (WorkspaceType.Incremental) { Identifier = $"{Stream.GetEscapedName()}+{AgentType.Workspace}"; } else { Identifier = GetDefaultWorkspaceIdentifier(Stream); } // Create the new workspace Workspace = new AgentWorkspace(WorkspaceType.Cluster, WorkspaceType.UserName, Identifier, WorkspaceType.Stream ?? Stream.Name, WorkspaceType.View, WorkspaceType.Incremental); return true; } } /// /// The escaped name of this stream. Removes all non-identifier characters. /// /// The stream object /// Escaped name for the stream public static string GetEscapedName(this IStream Stream) { return Regex.Replace(Stream.Name, @"[^a-zA-Z0-9_]", "+"); } /// /// Gets the default identifier for workspaces created for this stream. Just includes an escaped depot name. /// /// The stream object /// The default workspace identifier private static string GetDefaultWorkspaceIdentifier(IStream Stream) { return Regex.Replace(Stream.GetEscapedName(), @"^(\+\+[^+]*).*$", "$1"); } /// /// Checks the stream definition for consistency /// /// The stream object public static void Validate(this IStream Stream) { // Check the default preflight template is valid if (Stream.DefaultPreflight != null) { if (Stream.DefaultPreflight.TemplateRefId != null && !Stream.Templates.ContainsKey(Stream.DefaultPreflight.TemplateRefId.Value)) { throw new InvalidStreamException($"Default preflight template was listed as '{Stream.DefaultPreflight.TemplateRefId.Value}', but no template was found by that name"); } } // Check that all the templates are referenced by a tab HashSet RemainingTemplates = new HashSet(Stream.Templates.Keys); foreach(JobsTab JobsTab in Stream.Tabs.OfType()) { if (JobsTab.Templates != null) { RemainingTemplates.ExceptWith(JobsTab.Templates); } } if(RemainingTemplates.Count > 0) { throw new InvalidStreamException(String.Join("\n", RemainingTemplates.Select(x => $"Template '{x}' is not listed on any tab for {Stream.Id}"))); } // Check that all the agent types reference valid workspace names foreach (KeyValuePair Pair in Stream.AgentTypes) { string? WorkspaceTypeName = Pair.Value.Workspace; if (WorkspaceTypeName != null && !Stream.WorkspaceTypes.ContainsKey(WorkspaceTypeName)) { throw new InvalidStreamException($"Agent type '{Pair.Key}' references undefined workspace type '{Pair.Value.Workspace}' in {Stream.Id}"); } } } /// /// Converts to a public response object /// /// The stream object /// Whether to include the ACL in the response object /// The template refs for this stream. Passed separately because they have their own ACL. /// New response instance public static Api.GetStreamResponse ToApiResponse(this IStream Stream, bool bIncludeAcl, List ApiTemplateRefs) { List ApiTabs = Stream.Tabs.ConvertAll(x => x.ToResponse()); Dictionary ApiAgentTypes = Stream.AgentTypes.ToDictionary(x => x.Key, x => x.Value.ToApiResponse()); Dictionary ApiWorkspaceTypes = Stream.WorkspaceTypes.ToDictionary(x => x.Key, x => x.Value.ToApiResponse()); GetAclResponse? ApiAcl = (bIncludeAcl && Stream.Acl != null)? new GetAclResponse(Stream.Acl) : null; return new Api.GetStreamResponse(Stream.Id.ToString(), Stream.ProjectId.ToString(), Stream.Name, Stream.ConfigPath, Stream.ConfigRevision, Stream.Order, Stream.NotificationChannel, Stream.NotificationChannelFilter, Stream.TriageChannel, Stream.DefaultPreflight?.ToRequest(), ApiTabs, ApiAgentTypes, ApiWorkspaceTypes, ApiTemplateRefs, ApiAcl, Stream.PausedUntil, Stream.PauseComment); } /// /// Converts to an RPC response object /// /// The stream object /// New response instance public static HordeCommon.Rpc.GetStreamResponse ToRpcResponse(this IStream Stream) { HordeCommon.Rpc.GetStreamResponse Response = new HordeCommon.Rpc.GetStreamResponse(); Response.Name = Stream.Name; Response.AgentTypes.Add(Stream.AgentTypes.ToDictionary(x => x.Key, x => x.Value.ToRpcResponse())); Response.LastCommitTime = Stream.LastCommitTime.HasValue? Timestamp.FromDateTime(Stream.LastCommitTime.Value) : new Timestamp(); return Response; } /// /// Check if stream is paused for new builds /// /// The stream object /// Current time (allow tests to pass in a fake clock) /// If stream is paused public static bool IsPaused(this IStream Stream, DateTime CurrentTime) { return Stream.PausedUntil != null && Stream.PausedUntil > CurrentTime; } } /// /// Projection of a stream definition to just include permissions info /// public interface IStreamPermissions { /// /// ACL for the stream /// public Acl? Acl { get; } /// /// The project containing this stream /// public ProjectId ProjectId { get; } } }