// Copyright Epic Games, Inc. All Rights Reserved. using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; using EpicGames.Core; using Microsoft.Extensions.Logging; namespace UnrealBuildBase { /// /// FileHasher constructor /// /// Optional logger public class FileHasher(ILogger? logger = null) { private readonly struct CachedDigest(FileItem item, IoHash digest) { [JsonPropertyName("l")] public long Length { get; init; } = item.Length; [JsonPropertyName("w")] public long LastWriteTimeUtc { get; init; } = item.LastWriteTimeUtc.Ticks; [JsonPropertyName("d")] public IoHash Digest { get; init; } = digest; /// /// Check if a digest is up to date for a FileItem /// /// The FileItem to check /// If the digest is up to date public bool UpToDate(FileItem item) => item.Exists && Length == item.Length && LastWriteTimeUtc == item.LastWriteTimeUtc.Ticks; } // Dictionary containing cached digests, to prevent rehashing data if the file is unchanged ConcurrentDictionary CachedDigests = new(); /// /// Saves cached digests to disk. /// /// The location to save the digest data (json) public async Task Save(FileReference location) { using FileStream stream = FileReference.Open(location, FileMode.Create); await JsonSerializer.SerializeAsync(stream, new SortedDictionary(CachedDigests)); logger?.LogDebug("Saved {Entries} cached digest entries", CachedDigests.Count); } /// /// Loads cached digests from disk, replacing existing cache. /// /// The location to load the digest data from (json) public async Task Load(FileReference location) { if (!FileReference.Exists(location)) { return; } using FileStream stream = FileReference.Open(location, FileMode.Open, FileAccess.Read, FileShare.Read); ConcurrentDictionary? loadedDigests = await JsonSerializer.DeserializeAsync>(stream); if (loadedDigests != null) { CachedDigests = loadedDigests; logger?.LogDebug("Loaded {Entries} cached digest entries", CachedDigests.Count); } } /// /// Purges stale cached digests. /// public void PurgeStale() { int count = CachedDigests.Count; IEnumerable> valid = CachedDigests.Where((x) => x.Value.UpToDate(FileItem.GetItemByFileReference(FileReference.FromString(x.Key)))); CachedDigests = new(valid); logger?.LogDebug("Purged {Entries} stale cached digest entries", count - CachedDigests.Count); } /// /// Get the IoHash digest of a file's contents. /// /// The FileItem to digest /// Cancellation token for the operation /// A cache is maintained of digests and a file will not be rehashed if unchanged. /// The IoHash digest, or IoHash.Zero if the file doesn't exist. public async Task GetDigestAsync(FileItem item, CancellationToken cancellationToken = default) { if (CachedDigests.TryGetValue(item.FullName, out CachedDigest CachedHash)) { if (CachedHash.UpToDate(item)) { // Hash already calculated return CachedHash.Digest; } } if (!item.Exists) { return IoHash.Zero; } CachedDigest digest = await ComputeDigest(item, logger, cancellationToken); return CachedDigests.AddOrUpdate(item.FullName, digest, (key, oldValue) => digest).Digest; } /// /// In systems where the IoHash is already available, this allows it to be updated directly and /// not recomputed. /// /// The updated file item information /// The updated hash public void SetDigest(FileItem item, IoHash hash) { CachedDigest digest = new CachedDigest(item, hash); CachedDigests.AddOrUpdate(item.FullName, digest, (key, oldValue) => digest); } /// /// Get the IoHash digest of a file's contents. /// /// The FileReference to digest /// Cancellation token for the operation /// A cache is maintained of digests and a file will not be rehashed if unchanged. /// The IoHash digest, or IoHash.Zero if the file doesn't exist. public Task GetDigestAsync(FileReference location, CancellationToken cancellationToken = default) => GetDigestAsync(FileItem.GetItemByFileReference(location), cancellationToken); public IoHash GetDigest(FileItem item) => GetDigestAsync(item).Result; public IoHash GetDigest(FileReference location) => GetDigestAsync(location).Result; static async Task ComputeDigest(FileItem item, ILogger? logger, CancellationToken CancellationToken = default) { using FileStream stream = FileReference.Open(item.Location, FileMode.Open, FileAccess.Read, FileShare.Read); CachedDigest digest = new CachedDigest(item, await IoHash.ComputeAsync(stream, item.Length, CancellationToken)); logger?.LogDebug("Computed IoHash {Digest} File {Location} Size {Size} LastWrite {LastWrite}", digest.Digest, item.FullName, item.Length, item.LastWriteTimeUtc); return digest; } } }