// Copyright Epic Games, Inc. All Rights Reserved. using System.Collections.Generic; using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; using EpicGames.Horde.Storage; using Horde.Build.Agents.Sessions; using Horde.Build.Jobs; using Horde.Build.Server; using Horde.Build.Streams; using Horde.Build.Utilities; using MongoDB.Bson.Serialization.Attributes; using MongoDB.Driver; namespace Horde.Build.Logs { using JobId = ObjectId; using LogId = ObjectId; using SessionId = ObjectId; using StreamId = StringId; /// /// Wrapper around the jobs collection in a mongo DB /// public class LogFileCollection : ILogFileCollection { class LogChunkDocument : ILogChunk { public long Offset { get; set; } public int Length { get; set; } public int LineIndex { get; set; } [BsonIgnoreIfNull] public string? Server { get; set; } [BsonConstructor] public LogChunkDocument() { } public LogChunkDocument(LogChunkDocument other) { Offset = other.Offset; Length = other.Length; LineIndex = other.LineIndex; Server = other.Server; } public LogChunkDocument Clone() { return (LogChunkDocument)MemberwiseClone(); } } class LogFileDocument : ILogFile { [BsonRequired, BsonId] public LogId Id { get; set; } [BsonRequired] public JobId JobId { get; set; } public SessionId? SessionId { get; set; } public LogType Type { get; set; } [BsonIgnoreIfNull] public int? MaxLineIndex { get; set; } [BsonIgnoreIfNull] public long? IndexLength { get; set; } public List Chunks { get; set; } = new List(); public int LineCount { get; set; } public RefName RefName { get; set; } [BsonRequired] public int UpdateIndex { get; set; } IReadOnlyList ILogFile.Chunks => Chunks; [BsonConstructor] private LogFileDocument() { } public LogFileDocument(JobId jobId, SessionId? sessionId, LogType type, LogId? logId) { Id = logId ?? LogId.GenerateNewId(); JobId = jobId; SessionId = sessionId; Type = type; MaxLineIndex = 0; RefName = new RefName(Id.ToString()); } public LogFileDocument Clone() { LogFileDocument document = (LogFileDocument)MemberwiseClone(); document.Chunks = document.Chunks.ConvertAll(x => x.Clone()); return document; } } /// /// The jobs collection /// readonly IMongoCollection _logFiles; /// /// Hostname for the current server /// readonly string _hostName; /// /// Constructor /// /// The database service singleton public LogFileCollection(MongoService mongoService) { _logFiles = mongoService.GetCollection("LogFiles"); _hostName = Dns.GetHostName(); } /// public async Task CreateLogFileAsync(JobId jobId, SessionId? sessionId, LogType type, LogId? logId, CancellationToken cancellationToken) { LogFileDocument newLogFile = new (jobId, sessionId, type, logId); await _logFiles.InsertOneAsync(newLogFile, null, cancellationToken); return newLogFile; } /// public async Task UpdateLineCountAsync(ILogFile logFileInterface, int lineCount, CancellationToken cancellationToken) { FilterDefinition filter = Builders.Filter.Eq(x => x.Id, logFileInterface.Id); UpdateDefinition update = Builders.Update.Set(x => x.LineCount, lineCount).Inc(x => x.UpdateIndex, 1); return await _logFiles.FindOneAndUpdateAsync(filter, update, new FindOneAndUpdateOptions { ReturnDocument = ReturnDocument.After }, cancellationToken); } /// public async Task TryAddChunkAsync(ILogFile logFileInterface, long offset, int lineIndex, CancellationToken cancellationToken) { LogFileDocument logFile = ((LogFileDocument)logFileInterface).Clone(); int chunkIdx = logFile.Chunks.GetChunkForOffset(offset) + 1; LogChunkDocument chunk = new LogChunkDocument(); chunk.Offset = offset; chunk.LineIndex = lineIndex; chunk.Server = _hostName; logFile.Chunks.Insert(chunkIdx, chunk); UpdateDefinition update = Builders.Update.Set(x => x.Chunks, logFile.Chunks); if (chunkIdx == logFile.Chunks.Count - 1) { logFile.MaxLineIndex = null; update = update.Unset(x => x.MaxLineIndex); } if (!await TryUpdateLogFileAsync(logFile, update, cancellationToken)) { return null; } return logFile; } /// public async Task TryCompleteChunksAsync(ILogFile logFileInterface, IEnumerable chunkUpdates, CancellationToken cancellationToken) { LogFileDocument logFile = ((LogFileDocument)logFileInterface).Clone(); // Update the length of any complete chunks UpdateDefinitionBuilder updateBuilder = Builders.Update; List> updates = new List>(); foreach (CompleteLogChunkUpdate chunkUpdate in chunkUpdates) { LogChunkDocument chunk = logFile.Chunks[chunkUpdate.Index]; chunk.Length = chunkUpdate.Length; updates.Add(updateBuilder.Set(x => x.Chunks[chunkUpdate.Index].Length, chunkUpdate.Length)); if (chunkUpdate.Index == logFile.Chunks.Count - 1) { logFile.MaxLineIndex = chunk.LineIndex + chunkUpdate.LineCount; updates.Add(updateBuilder.Set(x => x.MaxLineIndex, logFile.MaxLineIndex)); } } // Try to apply the updates if (updates.Count > 0 && !await TryUpdateLogFileAsync(logFile, updateBuilder.Combine(updates), cancellationToken)) { return null; } return logFile; } /// public async Task TryUpdateIndexAsync(ILogFile logFileInterface, long newIndexLength, CancellationToken cancellationToken) { LogFileDocument logFile = ((LogFileDocument)logFileInterface).Clone(); UpdateDefinition update = Builders.Update.Set(x => x.IndexLength, newIndexLength); if (!await TryUpdateLogFileAsync(logFile, update, cancellationToken)) { return null; } logFile.IndexLength = newIndexLength; return logFile; } /// private async Task TryUpdateLogFileAsync(LogFileDocument current, UpdateDefinition update, CancellationToken cancellationToken) { int prevUpdateIndex = current.UpdateIndex; current.UpdateIndex++; UpdateResult result = await _logFiles.UpdateOneAsync(x => x.Id == current.Id && x.UpdateIndex == prevUpdateIndex, update.Set(x => x.UpdateIndex, current.UpdateIndex), cancellationToken: cancellationToken); return result.ModifiedCount == 1; } /// public async Task GetLogFileAsync(LogId logFileId, CancellationToken cancellationToken) { LogFileDocument logFile = await _logFiles.Find(x => x.Id == logFileId).FirstOrDefaultAsync(cancellationToken); return logFile; } /// public async Task> GetLogFilesAsync(int? index = null, int? count = null, CancellationToken cancellationToken = default) { IFindFluent query = _logFiles.Find(FilterDefinition.Empty); if(index != null) { query = query.Skip(index.Value); } if(count != null) { query = query.Limit(count.Value); } List results = await query.ToListAsync(cancellationToken); return results.ConvertAll(x => x); } } }