UnrealEngine/Engine/Source/Programs/Shared/EpicGames.Build/System/DirectoryItem.cs

// Copyright Epic Games, Inc. All Rights Reserved.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Security;
using EpicGames.Core;

namespace UnrealBuildBase
{
	/// <summary>
	/// Stores the state of a directory. May or may not exist.
	/// </summary>
	public class DirectoryItem : IComparable<DirectoryItem>, IEquatable<DirectoryItem>
	{
		/// <summary>
		/// Full path to the directory on disk
		/// </summary>
		public readonly DirectoryReference Location;

		/// <summary>
		/// Cached value for whether the directory exists
		/// </summary>
		Lazy<DirectoryInfo> Info;

		/// <summary>
		/// Cached maps of name to subdirectory and name to file
		/// </summary>
		Lazy<(Dictionary<string, DirectoryItem>, Dictionary<string, FileItem>)> Cache;

		/// <summary>
		/// Global map of location to item
		/// </summary>
		static ConcurrentDictionary<DirectoryReference, DirectoryItem> LocationToItem = [];

		/// <summary>
		/// Constructor
		/// </summary>
		/// <param name="Location">Path to this directory</param>
		/// <param name="Info">Information about this directory</param>
		private DirectoryItem(DirectoryReference Location, DirectoryInfo Info)
		{
			this.Location = Location;
			Cache = new(Scan);

			// For some reason we need to call an extra Refresh on Linux/Mac to not get wrong results from "Exists"
			this.Info = OperatingSystem.IsWindows()
				? new(Info)
				: new(() =>
				{
					Info.Refresh();
					return Info;
				});
		}

		/// <summary>
		/// Accessor for map of name to subdirectory item
		/// </summary>
		Dictionary<string, DirectoryItem> Directories => Cache.Value.Item1;

		/// <summary>
		/// Accessor for map of name to file
		/// </summary>
		Dictionary<string, FileItem> Files => Cache.Value.Item2;

		/// <summary>
		/// The name of this directory
		/// </summary>
		public string Name => Info.Value.Name;

		/// <summary>
		/// The full name of this directory
		/// </summary>
		public string FullName => Location.FullName;

		/// <summary>
		/// Whether the directory exists or not
		/// </summary>
		public bool Exists => Info.Value.Exists;

		/// <summary>
		/// The last write time of the file.
		/// </summary>
		public DateTime LastWriteTimeUtc => Info.Value.LastWriteTimeUtc;

		/// <summary>
		/// The creation time of the file.
		/// </summary>
		public DateTime CreationTimeUtc => Info.Value.CreationTimeUtc;

		/// <summary>
		/// Gets the parent directory item
		/// </summary>
		public DirectoryItem? GetParentDirectoryItem()
		{
			if (Info.Value.Parent == null)
			{
				return null;
			}
			else
			{
				return GetItemByDirectoryInfo(Info.Value.Parent);
			}
		}

		/// <summary>
		/// Gets a new directory item by combining the existing directory item with the given path fragments
		/// </summary>
		/// <param name="BaseDirectory">Base directory to append path fragments to</param>
		/// <param name="Fragments">The path fragments to append</param>
		/// <returns>Directory item corresponding to the combined path</returns>
		public static DirectoryItem Combine(DirectoryItem BaseDirectory, params string[] Fragments)
			=> GetItemByDirectoryReference(DirectoryReference.Combine(BaseDirectory.Location, Fragments));

		/// <summary>
		/// Finds or creates a directory item from its location
		/// </summary>
		/// <param name="Location">Path to the directory</param>
		/// <returns>The directory item for this location</returns>
		public static DirectoryItem GetItemByPath(string Location) => GetItemByDirectoryReference(new(Location));

		/// <summary>
		/// Finds or creates a directory item from its location
		/// </summary>
		/// <param name="Location">Path to the directory</param>
		/// <returns>The directory item for this location</returns>
		public static DirectoryItem GetItemByDirectoryReference(DirectoryReference Location)
			=> LocationToItem.TryGetValue(Location, out DirectoryItem? Result)
				? Result
				: LocationToItem.GetOrAdd(Location, new DirectoryItem(Location, new(Location.FullName)));

		/// <summary>
		/// Finds or creates a directory item from a DirectoryInfo object
		/// </summary>
		/// <param name="Info">Path to the directory</param>
		/// <returns>The directory item for this location</returns>
		public static DirectoryItem GetItemByDirectoryInfo(DirectoryInfo Info) => GetItemByDirectoryReference(new(Info));

		/// <summary>
		/// Reset the contents of the directory and allow them to be fetched again
		/// </summary>
		public void ResetCachedInfo()
		{
			Info = new(() =>
			{
				DirectoryInfo Info = Location.ToDirectoryInfo();
				Info.Refresh();
				return Info;
			});

			if (Cache.IsValueCreated)
			{
				(Dictionary<string, DirectoryItem> dirs, Dictionary<string, FileItem> files) = Cache.Value;
				foreach (DirectoryItem SubDirectory in dirs.Values)
				{
					SubDirectory.ResetCachedInfo();
				}
				foreach (FileItem File in files.Values)
				{
					File.ResetCachedInfo();
				}
				Cache = new(Scan);
			}
		}

		/// <summary>
		/// Resets the cached info, if the DirectoryInfo is not found don't create a new entry
		/// </summary>
		public static void ResetCachedInfo(string Path)
		{
			if (LocationToItem.TryGetValue(new DirectoryReference(Path), out DirectoryItem? Result))
			{
				Result.ResetCachedInfo();
			}
		}

		/// <summary>
		/// Resets all cached directory info. Significantly reduces performance; do not use unless strictly necessary.
		/// </summary>
		public static void ResetAllCachedInfo_SLOW()
		{
			LocationToItem.Values.AsParallel().ForAll(Item =>
			{
				Item.Info = new Lazy<DirectoryInfo>(() =>
				{
					DirectoryInfo Info = Item.Location.ToDirectoryInfo();
					Info.Refresh();
					return Info;
				});
				Item.Cache = new(Item.Scan);
			});
			FileItem.ResetAllCachedInfo_SLOW();
		}

		/// <summary>
		/// Caches the subdirectories of this directories
		/// </summary>
		public bool CacheDirectories() => Cache.Value.Item1 != null;

		/// <summary>
		/// Enumerates all the subdirectories
		/// </summary>
		/// <returns>Sequence of subdirectory items</returns>
		public IEnumerable<DirectoryItem> EnumerateDirectories()
		{
			CacheDirectories();
			return Directories.Values;
		}

		/// <summary>
		/// Attempts to get a sub-directory by name
		/// </summary>
		/// <param name="Name">Name of the directory</param>
		/// <param name="OutDirectory">If successful receives the matching directory item with this name</param>
		/// <returns>True if the file exists, false otherwise</returns>
		public bool TryGetDirectory(string Name, [NotNullWhen(true)] out DirectoryItem? OutDirectory)
		{
			if (Name.Length > 0 && Name[0] == '.')
			{
				if (Name.Length == 1)
				{
					OutDirectory = this;
					return true;
				}
				else if (Name.Length == 2 && Name[1] == '.')
				{
					OutDirectory = GetParentDirectoryItem();
					return OutDirectory != null;
				}
			}

			CacheDirectories();
			return Directories.TryGetValue(Name, out OutDirectory);
		}

		/// <summary>
		/// Scans the directory for directories and files, used for lazy initialization.
		/// </summary>
		/// <returns></returns>
		(Dictionary<string, DirectoryItem> , Dictionary<string, FileItem>) Scan()
		{
			try
			{
				// We want to turn enumerator to a list here since using EnumerateFiles.Count and then enumerate EnumerateFiles cause two iterations of findfirst/findnext
				// Also, we don't check if exists to minimize kernel call count
				List<FileSystemInfo> infos = Info.Value.EnumerateFileSystemInfos().ToList();

				int dirCount = 0;
				foreach (FileSystemInfo info in infos)
				{
					if (info.Attributes.HasFlag(FileAttributes.Directory))
					{
						++dirCount;
					}
				}

				Dictionary<string, DirectoryItem>? newDirs = new(dirCount, FileReference.Comparer);
				Dictionary<string, FileItem>? newFiles = new(infos.Count - dirCount, FileReference.Comparer);

				foreach (FileSystemInfo info in infos)
				{
					if (info.Attributes.HasFlag(FileAttributes.Directory))
					{
						newDirs.Add(info.Name, DirectoryItem.GetItemByDirectoryInfo((DirectoryInfo)info));
					}
					else
					{
						FileItem FileItem = FileItem.GetItemByFileInfo((FileInfo)info);

						// There are folders in linux sdk that has files with same name but different casing.
						// Ideally FileReference.Comparer should be case sensitive on linux/mac but I don't dare changing that right now
						if (newFiles.TryAdd(info.Name, FileItem))
						{
							FileItem.UpdateCachedDirectory(this);
						}
					}
				}

				return (newDirs, newFiles);
			}
			catch (DirectoryNotFoundException)
			{
			}
			catch (SecurityException)
			{
			}
			catch (UnauthorizedAccessException)
			{
			}

			return ([], []);
		}

		/// <summary>
		/// Caches the files in this directory
		/// </summary>
		public bool CacheFiles() => Cache.Value.Item2 != null;

		/// <summary>
		/// Enumerates all the files
		/// </summary>
		/// <returns>Sequence of FileItems</returns>
		public IEnumerable<FileItem> EnumerateFiles()
		{
			CacheFiles();
			return Files.Values;
		}

		/// <summary>
		/// Check if this directory contains any files
		/// </summary>
		/// <param name="searchOption">Directory search options</param>
		/// <returns>True if this directory has files</returns>
		public bool ContainsFiles(SearchOption searchOption = SearchOption.TopDirectoryOnly)
		{
			return searchOption == SearchOption.TopDirectoryOnly ? EnumerateFiles().Any(x => x.Exists) : (EnumerateFiles().Any(x => x.Exists) || EnumerateDirectories().Any(x => x.ContainsFiles(searchOption)));
		}

		/// <summary>
		/// Attempts to get a file from this directory by name. Unlike creating a file item and checking whether it exists, this will
		/// not create a permanent FileItem object if it does not exist.
		/// </summary>
		/// <param name="Name">Name of the file</param>
		/// <param name="OutFile">If successful receives the matching file item with this name</param>
		/// <returns>True if the file exists, false otherwise</returns>
		public bool TryGetFile(string Name, [NotNullWhen(true)] out FileItem? OutFile)
		{
			CacheFiles();
			return Files.TryGetValue(Name, out OutFile);
		}

		/// <summary>
		/// Formats this object as a string for debugging
		/// </summary>
		/// <returns>Location of the directory</returns>
		public override string ToString() => Location.ToString();

		/// <summary>
		/// Writes out all the enumerated files full names sorted to OutFile
		/// </summary>
		public static void WriteDebugFileWithAllEnumeratedFiles(string OutFile)
		{
			SortedSet<string> AllFiles = [];
			foreach (DirectoryItem Item in DirectoryItem.LocationToItem.Values)
			{
				if (Item.Files != null)
				{
					foreach (FileItem File in Item.EnumerateFiles())
					{
						AllFiles.Add(File.FullName);
					}
				}
			}
			File.WriteAllLines(OutFile, AllFiles);
		}

		#region IComparable, IEquatbale
		/// <inheritdoc/>
		public int CompareTo(DirectoryItem? other) => Location.CompareTo(other?.Location);

		/// <inheritdoc/>
		public bool Equals(DirectoryItem? other) => Location.Equals(other?.Location);

		/// <inheritdoc/>
		public override bool Equals(object? obj)
		{
			if (ReferenceEquals(this, obj))
			{
				return true;
			}

			if (obj is null)
			{
				return false;
			}

			return Equals(obj as DirectoryItem);
		}

		/// <inheritdoc/>
		public override int GetHashCode() => Location.GetHashCode();

		public static bool operator ==(DirectoryItem? left, DirectoryItem? right)
		{
			if (left is null)
			{
				return right is null;
			}

			return left.Equals(right);
		}

		public static bool operator !=(DirectoryItem? left, DirectoryItem? right)
		{
			return !(left == right);
		}

		public static bool operator <(DirectoryItem? left, DirectoryItem? right)
		{
			return left is null ? right is not null : left.CompareTo(right) < 0;
		}

		public static bool operator <=(DirectoryItem? left, DirectoryItem? right)
		{
			return left is null || left.CompareTo(right) <= 0;
		}

		public static bool operator >(DirectoryItem? left, DirectoryItem? right)
		{
			return left is not null && left.CompareTo(right) > 0;
		}

		public static bool operator >=(DirectoryItem? left, DirectoryItem? right)
		{
			return left is null ? right is null : left.CompareTo(right) >= 0;
		}
		#endregion
	}

	/// <summary>
	/// Helper functions for serialization
	/// </summary>
	public static class DirectoryItemExtensionMethods
	{
		/// <summary>
		/// Read a directory item from a binary archive
		/// </summary>
		/// <param name="Reader">Reader to serialize data from</param>
		/// <returns>Instance of the serialized directory item</returns>
		public static DirectoryItem? ReadDirectoryItem(this BinaryArchiveReader Reader)
		{
			// Use lambda that doesn't require anything to be captured thus eliminating an allocation.
			return Reader.ReadObjectReference(Reader => DirectoryItem.GetItemByDirectoryReference(Reader.ReadDirectoryReferenceNotNull()));
		}

		/// <summary>
		/// Write a directory item to a binary archive
		/// </summary>
		/// <param name="Writer">Writer to serialize data to</param>
		/// <param name="DirectoryItem">Directory item to write</param>
		public static void WriteDirectoryItem(this BinaryArchiveWriter Writer, DirectoryItem DirectoryItem)
		{
			// Use lambda that doesn't require anything to be captured thus eliminating an allocation.
			Writer.WriteObjectReference(DirectoryItem, (Writer, DirectoryItem) => Writer.WriteDirectoryReference(DirectoryItem.Location));
		}

		/// <summary>
		/// Writes a directory reference  to a binary archive
		/// </summary>
		/// <param name="Writer">The writer to output data to</param>
		/// <param name="Directory">The item to write</param>
		public static void WriteCompactDirectoryReference(this BinaryArchiveWriter Writer, DirectoryReference Directory)
		{
			DirectoryItem Item = DirectoryItem.GetItemByDirectoryReference(Directory);
			Writer.WriteDirectoryItem(Item);
		}

		/// <summary>
		/// Reads a directory reference from a binary archive
		/// </summary>
		/// <param name="Reader">Reader to serialize data from</param>
		/// <returns>New directory reference instance</returns>
		public static DirectoryReference ReadCompactDirectoryReference(this BinaryArchiveReader Reader)
		{
			return Reader.ReadDirectoryItem()!.Location;
		}
	}
}