// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using EpicGames.Core; using EpicGames.Perforce; using Horde.Server.Users; using Horde.Server.Utilities; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Horde.Server.Configuration { /// /// Source for reading config files /// public interface IConfigSource { /// /// URI scheme for this config source /// string Scheme { get; } /// /// Reads a config file from this source /// /// Locations of the config files to query /// Cancellation token for the operation /// Config file data Task GetAsync(Uri[] uris, CancellationToken cancellationToken); } /// /// Extension methods for /// public static class ConfigSource { /// /// Gets a single config file from a source /// /// Source to query /// Location of the config file to query /// Cancellation token for the operation /// Config file data public static async Task GetAsync(this IConfigSource source, Uri uri, CancellationToken cancellationToken) { IConfigFile[] result = await source.GetAsync(new[] { uri }, cancellationToken); return result[0]; } } /// /// In-memory config file source /// public sealed class InMemoryConfigSource : IConfigSource { class ConfigFileRevisionImpl : IConfigFile { public Uri Uri { get; } public string Revision { get; } public ReadOnlyMemory Data { get; } public IUser? Author => null; public ConfigFileRevisionImpl(Uri uri, string version, ReadOnlyMemory data) { Uri = uri; Revision = version; Data = data; } public ValueTask> ReadAsync(CancellationToken cancellationToken) => new ValueTask>(Data); } readonly Dictionary _files = new Dictionary(); /// /// Name of the scheme for this source /// public const string Scheme = "memory"; /// string IConfigSource.Scheme => Scheme; /// /// Manually adds a new config file /// /// Path to the config file /// Config file data public void Add(Uri path, ReadOnlyMemory data) { _files.Add(path, new ConfigFileRevisionImpl(path, "v1", data)); } /// public Task GetAsync(Uri[] uris, CancellationToken cancellationToken) { IConfigFile[] result = new IConfigFile[uris.Length]; for (int idx = 0; idx < uris.Length; idx++) { ConfigFileRevisionImpl? configFile; if (!_files.TryGetValue(uris[idx], out configFile)) { throw new FileNotFoundException($"Config file {uris[idx]} not found."); } result[idx] = configFile; } return Task.FromResult(result); } } /// /// Config file source which reads from the filesystem /// public sealed class FileConfigSource : IConfigSource { class ConfigFileImpl : IConfigFile { public Uri Uri { get; } public string Revision { get; } public DateTime LastWriteTimeUtc { get; } public ReadOnlyMemory Data { get; } public IUser? Author => null; public ConfigFileImpl(Uri uri, DateTime lastWriteTimeUtc, ReadOnlyMemory data) { Uri = uri; Revision = $"timestamp={lastWriteTimeUtc.Ticks}"; LastWriteTimeUtc = lastWriteTimeUtc; Data = data; } public ValueTask> ReadAsync(CancellationToken cancellationToken) => new ValueTask>(Data); } /// /// Name of the scheme for this source /// public const string Scheme = "file"; /// string IConfigSource.Scheme => Scheme; readonly DirectoryReference _baseDir; readonly ConcurrentDictionary _files = new ConcurrentDictionary(); /// /// Constructor /// public FileConfigSource() : this(DirectoryReference.GetCurrentDirectory()) { } /// /// Constructor /// /// Base directory for resolving relative paths public FileConfigSource(DirectoryReference baseDir) { _baseDir = baseDir; } /// public async Task GetAsync(Uri[] uris, CancellationToken cancellationToken) { IConfigFile[] files = new IConfigFile[uris.Length]; for (int idx = 0; idx < uris.Length; idx++) { Uri uri = uris[idx]; FileReference localPath = FileReference.Combine(_baseDir, uri.LocalPath); ConfigFileImpl? file; for (; ; ) { if (_files.TryGetValue(localPath, out file)) { if (FileReference.GetLastWriteTimeUtc(localPath) == file.LastWriteTimeUtc) { break; } else { _files.TryRemove(new KeyValuePair(localPath, file)); } } using (FileStream stream = FileReference.Open(localPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { using MemoryStream memoryStream = new MemoryStream(); await stream.CopyToAsync(memoryStream, cancellationToken); DateTime lastWriteTime = FileReference.GetLastWriteTimeUtc(localPath); file = new ConfigFileImpl(uri, lastWriteTime, memoryStream.ToArray()); } if (_files.TryAdd(localPath, file)) { break; } } files[idx] = file; } return files; } } /// /// Perforce cluster config file source /// public sealed class PerforceConfigSource : IConfigSource { class ConfigFileImpl : IConfigFile { public Uri Uri { get; } public int Change { get; } public string Revision { get; } public IUser? Author { get; } readonly PerforceConfigSource _owner; public ConfigFileImpl(Uri uri, int change, IUser? author, PerforceConfigSource owner) { Uri = uri; Change = change; Revision = $"{change}"; Author = author; _owner = owner; } public ValueTask> ReadAsync(CancellationToken cancellationToken) => _owner.ReadAsync(Uri, Change, cancellationToken); } /// /// Name of the scheme for this source /// public const string Scheme = "perforce"; /// string IConfigSource.Scheme => Scheme; readonly IOptionsMonitor _settings; readonly IUserCollection _userCollection; readonly IMemoryCache _cache; readonly ILogger _logger; /// /// Constructor /// public PerforceConfigSource(IOptionsMonitor settings, IUserCollection userCollection, IMemoryCache cache, ILogger logger) { _settings = settings; _userCollection = userCollection; _cache = cache; _logger = logger; } /// public async Task GetAsync(Uri[] uris, CancellationToken cancellationToken) { Dictionary results = new Dictionary(); foreach (IGrouping group in uris.GroupBy(x => x.Host)) { using (IPerforceConnection perforce = await ConnectAsync(group.Key, cancellationToken)) { FileSpecList fileSpec = group.Select(x => x.AbsolutePath).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); List records = await perforce.FStatAsync(FStatOptions.ShortenOutput, fileSpec, cancellationToken).ToListAsync(cancellationToken); records.RemoveAll(x => x.HeadAction == FileAction.Delete || x.HeadAction == FileAction.MoveDelete); Dictionary absolutePathToRecord = records.ToDictionary(x => x.DepotFile ?? String.Empty, x => x, StringComparer.OrdinalIgnoreCase); foreach (Uri uri in group) { FStatRecord? record; if (!absolutePathToRecord.TryGetValue(uri.AbsolutePath, out record)) { throw new FileNotFoundException($"Unable to read {uri}. No matching files found."); } IUser? author = await GetAuthorAsync(perforce, group.Key, record.HeadChange, cancellationToken); results[uri] = new ConfigFileImpl(uri, record.HeadChange, author, this); } } } return uris.ConvertAll(x => results[x]); } async ValueTask GetAuthorAsync(IPerforceConnection perforce, string host, int change, CancellationToken cancellationToken) { string cacheKey = $"{nameof(PerforceConfigSource)}:author:{host}@{change}"; if (!_cache.TryGetValue(cacheKey, out string? author)) { ChangeRecord record = await perforce.GetChangeAsync(GetChangeOptions.None, change, cancellationToken); using (ICacheEntry entry = _cache.CreateEntry(cacheKey)) { entry.SetSlidingExpiration(TimeSpan.FromHours(1.0)); entry.SetSize(256); entry.SetValue(record.User); } } return (author != null)? await _userCollection.FindUserByLoginAsync(author) : null; } async ValueTask> ReadAsync(Uri uri, int change, CancellationToken cancellationToken) { string cacheKey = $"{nameof(PerforceConfigSource)}:data:{uri}@{change}"; if (_cache.TryGetValue(cacheKey, out ReadOnlyMemory data)) { _logger.LogInformation("Read {Uri}@{Change} from cache ({Key})", uri, change, cacheKey); } else { _logger.LogInformation("Reading {Uri} at CL {Change} from Perforce", uri, change); using (IPerforceConnection perforce = await ConnectAsync(uri.Host, cancellationToken)) { PerforceResponse> response = await perforce.TryPrintDataAsync($"{uri.AbsolutePath}@{change}", cancellationToken); response.EnsureSuccess(); data = response.Data.Contents!; } using (ICacheEntry entry = _cache.CreateEntry(cacheKey)) { entry.SetSlidingExpiration(TimeSpan.FromHours(1.0)); entry.SetSize(data.Length); entry.SetValue(data); } } return data; } async Task ConnectAsync(string host, CancellationToken cancellationToken) { ServerSettings settings = _settings.CurrentValue; PerforceConnectionId connectionId = new PerforceConnectionId(); if (!String.IsNullOrEmpty(host)) { connectionId = new PerforceConnectionId(host); } PerforceConnectionSettings? connectionSettings = settings.Perforce.FirstOrDefault(x => x.Id == connectionId); if (connectionSettings == null) { if (connectionId == PerforceConnectionSettings.Default) { connectionSettings = new PerforceConnectionSettings(); } else { throw new InvalidOperationException($"No Perforce connection settings defined for '{connectionId}'."); } } IPerforceConnection connection = await PerforceConnection.CreateAsync(connectionSettings.ToPerforceSettings(), _logger); return connection; } } }