// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using EpicGames.Core;
using EpicGames.Perforce;
using Horde.Server.Users;
using Horde.Server.Utilities;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Horde.Server.Configuration
{
///
/// Source for reading config files
///
public interface IConfigSource
{
///
/// URI scheme for this config source
///
string Scheme { get; }
///
/// Reads a config file from this source
///
/// Locations of the config files to query
/// Cancellation token for the operation
/// Config file data
Task GetAsync(Uri[] uris, CancellationToken cancellationToken);
}
///
/// Extension methods for
///
public static class ConfigSource
{
///
/// Gets a single config file from a source
///
/// Source to query
/// Location of the config file to query
/// Cancellation token for the operation
/// Config file data
public static async Task GetAsync(this IConfigSource source, Uri uri, CancellationToken cancellationToken)
{
IConfigFile[] result = await source.GetAsync(new[] { uri }, cancellationToken);
return result[0];
}
}
///
/// In-memory config file source
///
public sealed class InMemoryConfigSource : IConfigSource
{
class ConfigFileRevisionImpl : IConfigFile
{
public Uri Uri { get; }
public string Revision { get; }
public ReadOnlyMemory Data { get; }
public IUser? Author => null;
public ConfigFileRevisionImpl(Uri uri, string version, ReadOnlyMemory data)
{
Uri = uri;
Revision = version;
Data = data;
}
public ValueTask> ReadAsync(CancellationToken cancellationToken) => new ValueTask>(Data);
}
readonly Dictionary _files = new Dictionary();
///
/// Name of the scheme for this source
///
public const string Scheme = "memory";
///
string IConfigSource.Scheme => Scheme;
///
/// Manually adds a new config file
///
/// Path to the config file
/// Config file data
public void Add(Uri path, ReadOnlyMemory data)
{
_files.Add(path, new ConfigFileRevisionImpl(path, "v1", data));
}
///
public Task GetAsync(Uri[] uris, CancellationToken cancellationToken)
{
IConfigFile[] result = new IConfigFile[uris.Length];
for (int idx = 0; idx < uris.Length; idx++)
{
ConfigFileRevisionImpl? configFile;
if (!_files.TryGetValue(uris[idx], out configFile))
{
throw new FileNotFoundException($"Config file {uris[idx]} not found.");
}
result[idx] = configFile;
}
return Task.FromResult(result);
}
}
///
/// Config file source which reads from the filesystem
///
public sealed class FileConfigSource : IConfigSource
{
class ConfigFileImpl : IConfigFile
{
public Uri Uri { get; }
public string Revision { get; }
public DateTime LastWriteTimeUtc { get; }
public ReadOnlyMemory Data { get; }
public IUser? Author => null;
public ConfigFileImpl(Uri uri, DateTime lastWriteTimeUtc, ReadOnlyMemory data)
{
Uri = uri;
Revision = $"timestamp={lastWriteTimeUtc.Ticks}";
LastWriteTimeUtc = lastWriteTimeUtc;
Data = data;
}
public ValueTask> ReadAsync(CancellationToken cancellationToken) => new ValueTask>(Data);
}
///
/// Name of the scheme for this source
///
public const string Scheme = "file";
///
string IConfigSource.Scheme => Scheme;
readonly DirectoryReference _baseDir;
readonly ConcurrentDictionary _files = new ConcurrentDictionary();
///
/// Constructor
///
public FileConfigSource()
: this(DirectoryReference.GetCurrentDirectory())
{
}
///
/// Constructor
///
/// Base directory for resolving relative paths
public FileConfigSource(DirectoryReference baseDir)
{
_baseDir = baseDir;
}
///
public async Task GetAsync(Uri[] uris, CancellationToken cancellationToken)
{
IConfigFile[] files = new IConfigFile[uris.Length];
for (int idx = 0; idx < uris.Length; idx++)
{
Uri uri = uris[idx];
FileReference localPath = FileReference.Combine(_baseDir, uri.LocalPath);
ConfigFileImpl? file;
for (; ; )
{
if (_files.TryGetValue(localPath, out file))
{
if (FileReference.GetLastWriteTimeUtc(localPath) == file.LastWriteTimeUtc)
{
break;
}
else
{
_files.TryRemove(new KeyValuePair(localPath, file));
}
}
using (FileStream stream = FileReference.Open(localPath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using MemoryStream memoryStream = new MemoryStream();
await stream.CopyToAsync(memoryStream, cancellationToken);
DateTime lastWriteTime = FileReference.GetLastWriteTimeUtc(localPath);
file = new ConfigFileImpl(uri, lastWriteTime, memoryStream.ToArray());
}
if (_files.TryAdd(localPath, file))
{
break;
}
}
files[idx] = file;
}
return files;
}
}
///
/// Perforce cluster config file source
///
public sealed class PerforceConfigSource : IConfigSource
{
class ConfigFileImpl : IConfigFile
{
public Uri Uri { get; }
public int Change { get; }
public string Revision { get; }
public IUser? Author { get; }
readonly PerforceConfigSource _owner;
public ConfigFileImpl(Uri uri, int change, IUser? author, PerforceConfigSource owner)
{
Uri = uri;
Change = change;
Revision = $"{change}";
Author = author;
_owner = owner;
}
public ValueTask> ReadAsync(CancellationToken cancellationToken) => _owner.ReadAsync(Uri, Change, cancellationToken);
}
///
/// Name of the scheme for this source
///
public const string Scheme = "perforce";
///
string IConfigSource.Scheme => Scheme;
readonly IOptionsMonitor _settings;
readonly IUserCollection _userCollection;
readonly IMemoryCache _cache;
readonly ILogger _logger;
///
/// Constructor
///
public PerforceConfigSource(IOptionsMonitor settings, IUserCollection userCollection, IMemoryCache cache, ILogger logger)
{
_settings = settings;
_userCollection = userCollection;
_cache = cache;
_logger = logger;
}
///
public async Task GetAsync(Uri[] uris, CancellationToken cancellationToken)
{
Dictionary results = new Dictionary();
foreach (IGrouping group in uris.GroupBy(x => x.Host))
{
using (IPerforceConnection perforce = await ConnectAsync(group.Key, cancellationToken))
{
FileSpecList fileSpec = group.Select(x => x.AbsolutePath).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
List records = await perforce.FStatAsync(FStatOptions.ShortenOutput, fileSpec, cancellationToken).ToListAsync(cancellationToken);
records.RemoveAll(x => x.HeadAction == FileAction.Delete || x.HeadAction == FileAction.MoveDelete);
Dictionary absolutePathToRecord = records.ToDictionary(x => x.DepotFile ?? String.Empty, x => x, StringComparer.OrdinalIgnoreCase);
foreach (Uri uri in group)
{
FStatRecord? record;
if (!absolutePathToRecord.TryGetValue(uri.AbsolutePath, out record))
{
throw new FileNotFoundException($"Unable to read {uri}. No matching files found.");
}
IUser? author = await GetAuthorAsync(perforce, group.Key, record.HeadChange, cancellationToken);
results[uri] = new ConfigFileImpl(uri, record.HeadChange, author, this);
}
}
}
return uris.ConvertAll(x => results[x]);
}
async ValueTask GetAuthorAsync(IPerforceConnection perforce, string host, int change, CancellationToken cancellationToken)
{
string cacheKey = $"{nameof(PerforceConfigSource)}:author:{host}@{change}";
if (!_cache.TryGetValue(cacheKey, out string? author))
{
ChangeRecord record = await perforce.GetChangeAsync(GetChangeOptions.None, change, cancellationToken);
using (ICacheEntry entry = _cache.CreateEntry(cacheKey))
{
entry.SetSlidingExpiration(TimeSpan.FromHours(1.0));
entry.SetSize(256);
entry.SetValue(record.User);
}
}
return (author != null)? await _userCollection.FindUserByLoginAsync(author) : null;
}
async ValueTask> ReadAsync(Uri uri, int change, CancellationToken cancellationToken)
{
string cacheKey = $"{nameof(PerforceConfigSource)}:data:{uri}@{change}";
if (_cache.TryGetValue(cacheKey, out ReadOnlyMemory data))
{
_logger.LogInformation("Read {Uri}@{Change} from cache ({Key})", uri, change, cacheKey);
}
else
{
_logger.LogInformation("Reading {Uri} at CL {Change} from Perforce", uri, change);
using (IPerforceConnection perforce = await ConnectAsync(uri.Host, cancellationToken))
{
PerforceResponse> response = await perforce.TryPrintDataAsync($"{uri.AbsolutePath}@{change}", cancellationToken);
response.EnsureSuccess();
data = response.Data.Contents!;
}
using (ICacheEntry entry = _cache.CreateEntry(cacheKey))
{
entry.SetSlidingExpiration(TimeSpan.FromHours(1.0));
entry.SetSize(data.Length);
entry.SetValue(data);
}
}
return data;
}
async Task ConnectAsync(string host, CancellationToken cancellationToken)
{
ServerSettings settings = _settings.CurrentValue;
PerforceConnectionId connectionId = new PerforceConnectionId();
if (!String.IsNullOrEmpty(host))
{
connectionId = new PerforceConnectionId(host);
}
PerforceConnectionSettings? connectionSettings = settings.Perforce.FirstOrDefault(x => x.Id == connectionId);
if (connectionSettings == null)
{
if (connectionId == PerforceConnectionSettings.Default)
{
connectionSettings = new PerforceConnectionSettings();
}
else
{
throw new InvalidOperationException($"No Perforce connection settings defined for '{connectionId}'.");
}
}
IPerforceConnection connection = await PerforceConnection.CreateAsync(connectionSettings.ToPerforceSettings(), _logger);
return connection;
}
}
}