// Copyright Epic Games, Inc. All Rights Reserved. using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; using EpicGames.Core; using Microsoft.Extensions.Logging; namespace UnrealBuildBase { public class FileHasher { private struct CachedDigest { [JsonPropertyName("l")] public long Length { get; init; } [JsonPropertyName("w")] public long LastWriteTimeUtc { get; init; } [JsonPropertyName("d")] public IoHash Digest { get; init; } public CachedDigest(FileItem item, IoHash digest) { Length = item.Length; LastWriteTimeUtc = item.LastWriteTimeUtc.Ticks; Digest = digest; } /// /// Check if a digest is up to date for a FileItem /// /// The FileItem to check /// If the digest is up to date public bool UpToDate(FileItem item) => item.Exists && Length == item.Length && LastWriteTimeUtc == item.LastWriteTimeUtc.Ticks; } // Dictionary containg cached digests, to prevent rehashing data if the file is unchanged ConcurrentDictionary CachedDigests = new(); ILogger? Logger; /// /// FileHasher constructor /// /// Optional logger public FileHasher(ILogger? logger = null) { Logger = logger; } /// /// Saves cached digests to disk. /// /// The location to save the digest data (json) public async Task Save(FileReference location) { using FileStream stream = FileReference.Open(location, FileMode.Create); await JsonSerializer.SerializeAsync(stream, new SortedDictionary(CachedDigests)); Logger?.LogDebug("Saved {Entries} cached digest entries", CachedDigests.Count); } /// /// Loads cached digests from disk, replacing existing cache. /// /// The location to load the digest data from (json) public async Task Load(FileReference location) { if (!FileReference.Exists(location)) { return; } using FileStream stream = FileReference.Open(location, FileMode.Open, FileAccess.Read, FileShare.Read); ConcurrentDictionary? loadedDigests = await JsonSerializer.DeserializeAsync>(stream); if (loadedDigests != null) { CachedDigests = loadedDigests; Logger?.LogDebug("Loaded {Entries} cached digest entries", CachedDigests.Count); } } /// /// Purges stale cached digests. /// public void PurgeStale() { int count = CachedDigests.Count; IEnumerable> valid = CachedDigests.Where((x) => x.Value.UpToDate(FileItem.GetItemByFileReference(FileReference.FromString(x.Key)))); CachedDigests = new(valid); Logger?.LogDebug("Purged {Entries} stale cached digest entries", count - CachedDigests.Count); } /// /// Get the IoHash digest of a file's contents. /// /// The FileItem to digest /// Cancellation token for the operation /// A cache is maintained of digests and a file will not be rehashed if unchanged. /// The IoHash digest, or IoHash.Zero if the file doesn't exist. public async Task GetDigestAsync(FileItem item, CancellationToken cancellationToken = default) { if (CachedDigests.TryGetValue(item.FullName, out CachedDigest CachedHash)) { if (CachedHash.UpToDate(item)) { // Hash already calculated return CachedHash.Digest; } } if (!item.Exists) { return IoHash.Zero; } CachedDigest digest = await ComputeDigest(item, Logger, cancellationToken); return CachedDigests.AddOrUpdate(item.FullName, digest, (key, oldValue) => digest).Digest; } /// /// Get the IoHash digest of a file's contents. /// /// The FileReference to digest /// Cancellation token for the operation /// A cache is maintained of digests and a file will not be rehashed if unchanged. /// The IoHash digest, or IoHash.Zero if the file doesn't exist. public Task GetDigestAsync(FileReference location, CancellationToken cancellationToken = default) => GetDigestAsync(FileItem.GetItemByFileReference(location), cancellationToken); public IoHash GetDigest(FileItem item) => GetDigestAsync(item).Result; public IoHash GetDigest(FileReference location) => GetDigestAsync(location).Result; static async Task ComputeDigest(FileItem item, ILogger? logger, CancellationToken CancellationToken = default) { using FileStream stream = FileReference.Open(item.Location, FileMode.Open, FileAccess.Read, FileShare.Read); CachedDigest digest = new CachedDigest(item, await IoHash.ComputeAsync(stream, CancellationToken)); logger?.LogDebug("Computed IoHash {Digest} File {Location} Size {Size} LastWrite {LastWrite}", digest.Digest, item.FullName, item.Length, item.LastWriteTimeUtc); return digest; } } }