923 lines
29 KiB
C++
923 lines
29 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "IndexedCacheStorageManager.h"
|
|
#include "IndexedCacheStorage.h"
|
|
|
|
#include "Async/Async.h"
|
|
#include "Async/Mutex.h"
|
|
#include "Async/UniqueLock.h"
|
|
#include "Experimental/Async/ConditionVariable.h"
|
|
#include "HAL/FileManager.h"
|
|
#include "HAL/IConsoleManager.h"
|
|
#include "HAL/PlatformMisc.h"
|
|
#include "Misc/ConfigCacheIni.h"
|
|
#include "HAL/PlatformStackWalk.h"
|
|
#include "Misc/ScopeExit.h"
|
|
#include "Misc/ScopeRWLock.h"
|
|
#include "String/LexFromString.h"
|
|
#include "String/ParseTokens.h"
|
|
|
|
DECLARE_LOG_CATEGORY_EXTERN(LogIndexedCacheStorageManager, Display, All);
|
|
DEFINE_LOG_CATEGORY(LogIndexedCacheStorageManager);
|
|
#define LOG_ICS(Verbosity, STRING, ...) UE_LOG(LogIndexedCacheStorageManager, Verbosity, TEXT("%s: ") STRING, ANSI_TO_TCHAR(__FUNCTION__), ##__VA_ARGS__ )
|
|
|
|
namespace Experimental
|
|
{
|
|
|
|
class FCacheStorageMetaData
|
|
{
|
|
public:
|
|
enum class ECacheStorageState : uint8
|
|
{
|
|
Deleted,
|
|
Created,
|
|
Mounted,
|
|
Count
|
|
};
|
|
|
|
const TCHAR* LexToString(ECacheStorageState Val)
|
|
{
|
|
static const TCHAR* Strings[] =
|
|
{
|
|
TEXT("ECacheStorageState::Deleted"),
|
|
TEXT("ECacheStorageState::Created"),
|
|
TEXT("ECacheStorageState::Mounted"),
|
|
};
|
|
static_assert(int32(ECacheStorageState::Count) == UE_ARRAY_COUNT(Strings), "");
|
|
|
|
return Strings[int32(Val)];
|
|
}
|
|
|
|
FCacheStorageMetaData(int32 InCacheIndex, const FString& InMountName, bool bCacheExists, uint64 InEarlyStartupSize)
|
|
: MountName(InMountName)
|
|
, CacheIndex(InCacheIndex)
|
|
, EarlyStartupSize(InEarlyStartupSize)
|
|
{
|
|
CacheStorageState = bCacheExists ? ECacheStorageState::Created : ECacheStorageState::Deleted;
|
|
}
|
|
|
|
bool IsValid() const
|
|
{
|
|
return !MountName.IsEmpty();
|
|
}
|
|
|
|
const FString& GetMountName() const
|
|
{
|
|
return MountName;
|
|
}
|
|
|
|
uint64 GetEarlyStartupSize() const
|
|
{
|
|
return EarlyStartupSize;
|
|
}
|
|
|
|
FString GetMountPath() const
|
|
{
|
|
FString MountedPath;
|
|
IIndexedCacheStorage::Get().GetCacheStorageMountPath(MountedPath, MountName);
|
|
return MountedPath;
|
|
}
|
|
|
|
void WaitForPredicateWhileLocked(const TFunction<bool()>& Predicate)
|
|
{
|
|
while (!Predicate())
|
|
{
|
|
ConditionVariable.Wait(Mutex);
|
|
}
|
|
}
|
|
|
|
void NotifyWaitersWhileLocked()
|
|
{
|
|
ConditionVariable.NotifyAll();
|
|
}
|
|
|
|
bool CreateCacheStorage(uint64 RequestNumberOfBytes)
|
|
{
|
|
UE::TUniqueLock Lock(Mutex);
|
|
|
|
check(EarlyStartupSize == 0 || RequestNumberOfBytes == EarlyStartupSize);
|
|
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("CreateCacheStorage request for %ull bytes at cache %d. Current state %s"), RequestNumberOfBytes, CacheIndex, LexToString(CacheStorageState));
|
|
|
|
WaitForPredicateWhileLocked([this]()->bool
|
|
{
|
|
return CacheStorageState != ECacheStorageState::Mounted;
|
|
});;
|
|
|
|
check(MountRefCount == 0);
|
|
|
|
// Do not uncomment the code below: even if the cache is in 'Created' state, we might want to handle the case of a reallocation and
|
|
// the lower level takes care of either reuse the one we asked for (same size), or destroy and create a new one
|
|
//if (CacheStorageState == ECacheStorageState::Created)
|
|
//{
|
|
// return true;
|
|
//}
|
|
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("CreateCacheStorage creating cache %d with %ull bytes"), CacheIndex, RequestNumberOfBytes);
|
|
bool bCacheExists = IIndexedCacheStorage::Get().CreateCacheStorage(RequestNumberOfBytes, CacheIndex);
|
|
|
|
if (bCacheExists)
|
|
{
|
|
CacheStorageState = ECacheStorageState::Created;
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("CreateCacheStorage created cache %d at %s. Current state %s"), CacheIndex, *MountName, LexToString(CacheStorageState));
|
|
}
|
|
else
|
|
{
|
|
CacheStorageState = ECacheStorageState::Deleted;
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("CreateCacheStorage failed to create cache %d at %s. Current state %s"), CacheIndex, *MountName, LexToString(CacheStorageState));
|
|
}
|
|
|
|
NotifyWaitersWhileLocked();
|
|
|
|
return bCacheExists;
|
|
}
|
|
|
|
void DestroyCacheStorage()
|
|
{
|
|
UE::TUniqueLock Lock(Mutex);
|
|
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("DestroyCacheStorage request at cache %d. Current state %s"), CacheIndex, LexToString(CacheStorageState));
|
|
|
|
WaitForPredicateWhileLocked([this]()->bool
|
|
{
|
|
return CacheStorageState != ECacheStorageState::Mounted;
|
|
});;
|
|
|
|
check(MountRefCount == 0);
|
|
|
|
if (CacheStorageState == ECacheStorageState::Deleted)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("DestroyCacheStorage skipped deletion of cache %d at %s"), CacheIndex, *MountName);
|
|
return;
|
|
}
|
|
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("DestroyCacheStorage deleting cache %d"), CacheIndex);
|
|
bool bCachedDestroyed = IIndexedCacheStorage::Get().DestroyCacheStorage(CacheIndex);
|
|
|
|
if (bCachedDestroyed)
|
|
{
|
|
CacheStorageState = ECacheStorageState::Deleted;
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("DestroyCacheStorage deleted cache %d at %s. Current state %s"), CacheIndex, *MountName, LexToString(CacheStorageState));
|
|
}
|
|
else
|
|
{
|
|
CacheStorageState = ECacheStorageState::Created;
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("DestroyCacheStorage failed to delete cache %d at %s. Current state %s"), CacheIndex, *MountName, LexToString(CacheStorageState));
|
|
}
|
|
|
|
NotifyWaitersWhileLocked();
|
|
}
|
|
|
|
uint64 GetCacheStorageInfo()
|
|
{
|
|
UE::TUniqueLock Lock(Mutex);
|
|
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("GetCacheStorageInfo request at cache %d. Current state %s"), CacheIndex, LexToString(CacheStorageState));
|
|
|
|
uint64 CurrentDataSize = 0;
|
|
uint64 CurrentJournalSize = 0;
|
|
|
|
if (CacheStorageState == ECacheStorageState::Deleted)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("GetCacheStorageInfo cache %d is empty"), CacheIndex, CurrentDataSize);
|
|
return CurrentDataSize;
|
|
}
|
|
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("GetCacheStorageInfo cache %d"), CacheIndex);
|
|
IIndexedCacheStorage::Get().GetCacheStorageInfo(CacheIndex, CurrentDataSize, CurrentJournalSize);
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("GetCacheStorageInfo cache %d = data %llu bytes / journal %llu bytes"), CacheIndex, CurrentDataSize, CurrentJournalSize);
|
|
|
|
NotifyWaitersWhileLocked();
|
|
|
|
return CurrentDataSize;
|
|
}
|
|
|
|
FString MountCacheStorage()
|
|
{
|
|
UE::TUniqueLock Lock(Mutex);
|
|
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("MountCacheStorage request at cache %d. Current state %s"), CacheIndex, LexToString(CacheStorageState));
|
|
|
|
if (CacheStorageState == ECacheStorageState::Deleted)
|
|
{
|
|
check(MountRefCount == 0);
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("MountCacheStorage skip mount for cache %d deleted"), CacheIndex);
|
|
return FString();
|
|
}
|
|
|
|
FString MountedPath = GetMountPath();
|
|
if (CacheStorageState == ECacheStorageState::Mounted)
|
|
{
|
|
check(MountRefCount > 0);
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("MountCacheStorage skip mount for cache %d already mounted (%d)"), CacheIndex, MountRefCount);
|
|
++MountRefCount;
|
|
return MountedPath;
|
|
}
|
|
|
|
check(MountRefCount == 0);
|
|
check(CacheStorageState == ECacheStorageState::Created);
|
|
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("MountCacheStorage mounting cache %d at %s"), CacheIndex, *MountedPath);
|
|
bool bMounted = IIndexedCacheStorage::Get().MountCacheStorage(MountedPath, CacheIndex, MountName);
|
|
|
|
check(MountRefCount == 0);
|
|
|
|
if (bMounted)
|
|
{
|
|
CacheStorageState = ECacheStorageState::Mounted;
|
|
MountRefCount = 1;
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("MountCacheStorage mounted cache %d at %s. Current state %s"), CacheIndex, *MountedPath, LexToString(CacheStorageState));
|
|
}
|
|
else
|
|
{
|
|
CacheStorageState = ECacheStorageState::Created;
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("MountCacheStorage failed to mount cache %d at %s. Current state %s"), CacheIndex, *MountedPath, LexToString(CacheStorageState));
|
|
}
|
|
|
|
NotifyWaitersWhileLocked();
|
|
return (bMounted) ? MountedPath : FString();
|
|
}
|
|
|
|
|
|
void UnmountCacheStorage()
|
|
{
|
|
UE::TUniqueLock Lock(Mutex);
|
|
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("UnmountCacheStorage request at cache %d. Current state %s"), CacheIndex, LexToString(CacheStorageState));
|
|
|
|
if (CacheStorageState == ECacheStorageState::Deleted || CacheStorageState == ECacheStorageState::Created)
|
|
{
|
|
check(MountRefCount == 0);
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("UnmountCacheStorage skip unmount for cache %d already unmounted"), CacheIndex);
|
|
return;
|
|
}
|
|
|
|
check(CacheStorageState == ECacheStorageState::Mounted);
|
|
check(MountRefCount > 0);
|
|
|
|
if (MountRefCount > 1)
|
|
{
|
|
--MountRefCount;
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("UnmountCacheStorage cache %d still mounted (%d)"), CacheIndex, MountRefCount);
|
|
return;
|
|
}
|
|
|
|
FString MountedPath = GetMountPath();
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("UnmountCacheStorage unmounting cache %d at %s"), CacheIndex, *MountedPath);
|
|
IIndexedCacheStorage::Get().UnmountCacheStorage(*MountName);
|
|
CacheStorageState = ECacheStorageState::Created;
|
|
UE_LOG(LogIndexedCacheStorageManager, Display, TEXT("UnmountCacheStorage unmounted cache %d at %s. Current state %s"), CacheIndex, *MountedPath, LexToString(CacheStorageState));
|
|
|
|
check(MountRefCount == 1);
|
|
MountRefCount = 0;
|
|
NotifyWaitersWhileLocked();
|
|
}
|
|
|
|
private:
|
|
const FString MountName;
|
|
const int32 CacheIndex;
|
|
const uint64 EarlyStartupSize;
|
|
int32 MountRefCount = 0;
|
|
ECacheStorageState CacheStorageState = ECacheStorageState::Deleted;
|
|
UE::FMutex Mutex;
|
|
UE::FConditionVariable ConditionVariable;
|
|
|
|
};
|
|
|
|
using FCacheStorageMetaDataArray = TArray<TUniquePtr<FCacheStorageMetaData> >;
|
|
|
|
struct FIndexedCacheStorageMetaData
|
|
{
|
|
FIndexedCacheStorageMetaData(FCacheStorageMetaDataArray&& InCacheStorageMetaDataArray)
|
|
: IndexedCacheEntriesMetaData(MoveTemp(InCacheStorageMetaDataArray))
|
|
{
|
|
}
|
|
|
|
const FCacheStorageMetaDataArray IndexedCacheEntriesMetaData;
|
|
|
|
};
|
|
|
|
static FIndexedCacheStorageManager* GIndexedCacheStorageManager = nullptr;
|
|
|
|
FIndexedCacheStorageManager& FIndexedCacheStorageManager::Get()
|
|
{
|
|
static FIndexedCacheStorageManager Instance;
|
|
UE_CALL_ONCE([&]
|
|
{
|
|
GIndexedCacheStorageManager = &Instance;
|
|
}
|
|
);
|
|
|
|
return Instance;
|
|
}
|
|
|
|
void FIndexedCacheStorageManager::EnsureEarlyStartupCache(const TArray<bool>& InCacheStorageAlreadyExists)
|
|
{
|
|
int32 EarlyStartupCacheIndex = GetStorageIndex(TEXT("EarlyStartupCache"));
|
|
if (EarlyStartupCacheIndex == 0)
|
|
{
|
|
LOG_ICS(Display, TEXT("No Storage named 'EarlyStartupCache' has been set. Skipping"));
|
|
return;
|
|
}
|
|
|
|
TArray<bool> CacheStorageAlreadyExists(InCacheStorageAlreadyExists);
|
|
|
|
if (CacheStorageAlreadyExists[EarlyStartupCacheIndex])
|
|
{
|
|
LOG_ICS(Error, TEXT("'EarlyStartupCache' entry already exists. Destroying it"));
|
|
DestroyCacheStorage(EarlyStartupCacheIndex);
|
|
CacheStorageAlreadyExists[EarlyStartupCacheIndex] = false;
|
|
}
|
|
|
|
TMap<int32, uint64> EarlyStartupCaches;
|
|
{
|
|
uint64 PersistentDownloadDirEarlyStartupSize = IIndexedCacheStorage::Get().GetPersistentDownloadDirEarlyStartupSize();
|
|
if (PersistentDownloadDirEarlyStartupSize > 0)
|
|
{
|
|
EarlyStartupCaches.Add(0, PersistentDownloadDirEarlyStartupSize);
|
|
}
|
|
|
|
for (int32 CacheIndex = 0; CacheIndex < IndexedCacheStorageMetaData->IndexedCacheEntriesMetaData.Num(); ++CacheIndex)
|
|
{
|
|
FCacheStorageMetaData* CacheStorageMetaData = IndexedCacheStorageMetaData->IndexedCacheEntriesMetaData[CacheIndex].Get();
|
|
if (CacheStorageMetaData == nullptr)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
uint64 EarlyStartupSize = CacheStorageMetaData->GetEarlyStartupSize();
|
|
|
|
if (EarlyStartupSize > 0)
|
|
{
|
|
EarlyStartupCaches.Add(CacheIndex, EarlyStartupSize);
|
|
}
|
|
}
|
|
}
|
|
|
|
uint64 EarlyStartupCacheSize = 0;
|
|
uint64 NumCacheStorageInGroup = 0;
|
|
|
|
for (const TPair<int32, uint64>& EarlyStartupCache : EarlyStartupCaches)
|
|
{
|
|
uint64 CacheStorageSizeForCommonAlloc = 0;
|
|
const int32 CacheStorageIndex = EarlyStartupCache.Key;
|
|
const uint64 CacheStorageNewDataSize = EarlyStartupCache.Value;
|
|
const uint64 CacheStorageNewJournalSize = IIndexedCacheStorage::Get().GetCacheStorageJournalSize(CacheStorageNewDataSize, CacheStorageIndex);
|
|
if (CacheStorageAlreadyExists[CacheStorageIndex])
|
|
{
|
|
uint64 CacheStorageCurrentDataSize = 0;
|
|
uint64 CacheStorageCurrentJournalSize = 0;
|
|
IIndexedCacheStorage::Get().GetCacheStorageInfo(CacheStorageIndex, CacheStorageCurrentDataSize, CacheStorageCurrentJournalSize);
|
|
if (CacheStorageCurrentDataSize != CacheStorageNewDataSize || CacheStorageCurrentJournalSize != CacheStorageNewJournalSize)
|
|
{
|
|
LOG_ICS(Display, TEXT("Cache %d size mismatch: data: %llu vs %llu. journal: %llu vs %llu. Destroying cache"),
|
|
CacheStorageIndex, CacheStorageCurrentDataSize, CacheStorageNewDataSize, CacheStorageCurrentJournalSize, CacheStorageNewJournalSize);
|
|
|
|
DestroyCacheStorage(CacheStorageIndex);
|
|
CacheStorageSizeForCommonAlloc = CacheStorageNewDataSize + CacheStorageNewJournalSize;
|
|
}
|
|
else
|
|
{
|
|
LOG_ICS(Display, TEXT("Cache %d up to date with %llu bytes. Skipping from grouped alloc"), CacheStorageIndex, CacheStorageCurrentDataSize);
|
|
}
|
|
}
|
|
else if (CacheStorageNewDataSize > 0)
|
|
{
|
|
LOG_ICS(Display, TEXT("Cache %d does not exist, adding %llu bytes"), CacheStorageIndex, CacheStorageNewDataSize);
|
|
CacheStorageSizeForCommonAlloc = CacheStorageNewDataSize + CacheStorageNewJournalSize;
|
|
}
|
|
|
|
if (CacheStorageSizeForCommonAlloc > 0)
|
|
{
|
|
EarlyStartupCacheSize += CacheStorageSizeForCommonAlloc;
|
|
++NumCacheStorageInGroup;
|
|
}
|
|
}
|
|
|
|
if (EarlyStartupCacheSize == 0)
|
|
{
|
|
LOG_ICS(Display, TEXT("All caches are up-to-date, skipping reservation"));
|
|
return;
|
|
}
|
|
|
|
if (NumCacheStorageInGroup <= 1)
|
|
{
|
|
LOG_ICS(Display, TEXT("Only one cache requested, skipping reservation"));
|
|
return;
|
|
}
|
|
|
|
{
|
|
// Make sure we alloc at least 2MB
|
|
const uint64 MinGroupAllocSize = 2 << 20;
|
|
EarlyStartupCacheSize = FMath::Max(MinGroupAllocSize, EarlyStartupCacheSize);
|
|
|
|
// Remove alloc meta data because we want to allocate exactly EarlyStartupCacheSize
|
|
uint64 EarlyStartupCacheSizeMetaData = IIndexedCacheStorage::Get().GetCacheStorageJournalSize(EarlyStartupCacheSize, EarlyStartupCacheIndex);
|
|
// make sure we don't underflow
|
|
EarlyStartupCacheSize -= FMath::Min(EarlyStartupCacheSizeMetaData, EarlyStartupCacheSize);
|
|
|
|
// Make sure we alloc at least the metadata size
|
|
EarlyStartupCacheSize = FMath::Max(EarlyStartupCacheSizeMetaData, EarlyStartupCacheSize);
|
|
}
|
|
|
|
LOG_ICS(Display, TEXT("Ensure %llu bytes through cache %d"), EarlyStartupCacheSize, EarlyStartupCacheIndex);
|
|
bool bCacheCreated = true;
|
|
while (true)
|
|
{
|
|
if (CreateCacheStorage(EarlyStartupCacheSize, EarlyStartupCacheIndex))
|
|
{
|
|
DestroyCacheStorage(EarlyStartupCacheIndex);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
uint64 ParseSizeParam(FStringView Value)
|
|
{
|
|
Value = Value.TrimStartAndEnd();
|
|
|
|
uint64 Size = 0;
|
|
LexFromString(Size, Value);
|
|
if (Size > 0)
|
|
{
|
|
if (Value.EndsWith(TEXT("GB"))) return Size << 30;
|
|
if (Value.EndsWith(TEXT("MB"))) return Size << 20;
|
|
if (Value.EndsWith(TEXT("KB"))) return Size << 10;
|
|
}
|
|
return Size;
|
|
}
|
|
|
|
// Parse IniString expected to be in the form of OutIniName:[OutIniSection]:OutIniKey
|
|
bool GetIniParameters(FString& OutIniName, FString& OutIniSection, FString& OutIniKey, const FString& IniString)
|
|
{
|
|
TArray<FStringView> ConfigStringArrayNameSize;
|
|
UE::String::ParseTokens(IniString, TEXTVIEW(":"), [&](FStringView Element)
|
|
{
|
|
ConfigStringArrayNameSize.Add(Element);
|
|
});
|
|
|
|
if (ConfigStringArrayNameSize.Num() != 3 ||
|
|
ConfigStringArrayNameSize[0].Len() == 0 ||
|
|
ConfigStringArrayNameSize[1].Len() == 0 ||
|
|
ConfigStringArrayNameSize[2].Len() == 0 )
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("Error while parsing %s: expecting OutIniName:[OutIniSection]:OutIniKey"), *IniString);
|
|
return false;
|
|
}
|
|
|
|
FStringView IniSectionView = ConfigStringArrayNameSize[1];
|
|
if (IniSectionView[0] != TEXT('[') || IniSectionView[IniSectionView.Len()-1] != TEXT(']'))
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("Error while parsing ini section %.*s"), IniSectionView.Len(), IniSectionView.GetData());
|
|
return false;
|
|
}
|
|
|
|
IniSectionView.RemovePrefix(1);
|
|
IniSectionView.RemoveSuffix(1);
|
|
|
|
OutIniName = ConfigStringArrayNameSize[0];
|
|
OutIniSection = IniSectionView;
|
|
OutIniKey = ConfigStringArrayNameSize[2];
|
|
|
|
return true;
|
|
}
|
|
|
|
void FIndexedCacheStorageManager::Initialize()
|
|
{
|
|
uint32 NumIndexedCacheEntries = IIndexedCacheStorage::Get().GetNumIndexedCacheStorages();
|
|
if (NumIndexedCacheEntries == 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
FCacheStorageMetaDataArray IndexedCacheEntriesMetaData;
|
|
IndexedCacheEntriesMetaData.SetNum(NumIndexedCacheEntries);
|
|
|
|
TArray<int32> CacheStorageIndices;
|
|
IIndexedCacheStorage::Get().EnumerateCacheStorages(CacheStorageIndices);
|
|
|
|
TArray<bool> CacheStorageAlreadyExists;
|
|
CacheStorageAlreadyExists.SetNumZeroed(NumIndexedCacheEntries);
|
|
|
|
for (int32 CacheStorageIndex : CacheStorageIndices)
|
|
{
|
|
CacheStorageAlreadyExists[CacheStorageIndex] = true;
|
|
}
|
|
|
|
|
|
// Takes on the pattern
|
|
// (Name="CacheStorageName",Index=CacheStorageIndex,Mount="CacheStorageMount")
|
|
TArray<FString> StorageConfigs;
|
|
GConfig->GetArray(TEXT("IndexedCacheStorage"), TEXT("Storage"), StorageConfigs, GEngineIni);
|
|
|
|
uint32 RegisteredStorages = 0;
|
|
for (const FString& Category : StorageConfigs)
|
|
{
|
|
FString TrimmedCategory = Category;
|
|
TrimmedCategory.TrimStartAndEndInline();
|
|
if (TrimmedCategory.Left(1) == TEXT("("))
|
|
{
|
|
TrimmedCategory.RightChopInline(1, EAllowShrinking::No);
|
|
}
|
|
if (TrimmedCategory.Right(1) == TEXT(")"))
|
|
{
|
|
TrimmedCategory.LeftChopInline(1, EAllowShrinking::No);
|
|
}
|
|
|
|
// Find all custom chunks and parse
|
|
const TCHAR* PropertyName = TEXT("Name=");
|
|
const TCHAR* PropertyIndex = TEXT("Index=");
|
|
const TCHAR* PropertyMount = TEXT("Mount=");
|
|
const TCHAR* PropertyEarlyStartup = TEXT("EarlyStartup=");
|
|
FString StorageName;
|
|
int32 StorageIndex;
|
|
FString StorageMount;
|
|
|
|
if (FParse::Value(*TrimmedCategory, PropertyName, StorageName) &&
|
|
FParse::Value(*TrimmedCategory, PropertyIndex, StorageIndex) &&
|
|
FParse::Value(*TrimmedCategory, PropertyMount, StorageMount))
|
|
{
|
|
if (StorageName.Len() == 0)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("Found empty Name in [IndexedCacheStorage]:Storage"));
|
|
continue;
|
|
}
|
|
|
|
if (StorageIndex <= 0 || StorageIndex >= (int32)NumIndexedCacheEntries)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("Found invalid Index in [IndexedCacheStorage]:Storage: %d, max allowed = %d"), StorageIndex, NumIndexedCacheEntries - 1);
|
|
continue;
|
|
}
|
|
|
|
if (StorageMount.Len() == 0)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("Found empty Mount in [IndexedCacheStorage]:Storage"));
|
|
continue;
|
|
}
|
|
|
|
StorageName.ReplaceInline(TEXT("\""), TEXT(""));
|
|
StorageMount.ReplaceInline(TEXT("\""), TEXT(""));
|
|
|
|
int32& ExistingStorageIndex = StorageNameToStorageIndex.FindOrAdd(StorageName, 0);
|
|
if (ExistingStorageIndex > 0)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("Found an existing entry with name %s in [IndexedCacheStorage]:Storage"), *StorageName);
|
|
continue;
|
|
}
|
|
ExistingStorageIndex = StorageIndex;
|
|
|
|
uint64 EarlyStartupSize = 0;
|
|
{
|
|
FString EarlyStartupIniPath;
|
|
if (FParse::Value(*TrimmedCategory, PropertyEarlyStartup, EarlyStartupIniPath))
|
|
{
|
|
FString IniName;
|
|
FString IniSection;
|
|
FString IniKey;
|
|
if (GetIniParameters(IniName, IniSection, IniKey, EarlyStartupIniPath))
|
|
{
|
|
FString EarlyStartupSizeStr = GConfig->GetStr(*IniSection, *IniKey, *IniName);
|
|
EarlyStartupSize = ParseSizeParam(EarlyStartupSizeStr);
|
|
if (EarlyStartupSize == 0)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error,
|
|
TEXT("%s%s specified, but no %s:%s exists in the %s ini file"), PropertyEarlyStartup, *EarlyStartupIniPath, *IniSection, *IniKey, *IniName);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (IndexedCacheEntriesMetaData[StorageIndex] == nullptr)
|
|
{
|
|
IndexedCacheEntriesMetaData[StorageIndex] = MakeUnique<FCacheStorageMetaData>(StorageIndex, StorageMount, CacheStorageAlreadyExists[StorageIndex], EarlyStartupSize);
|
|
++RegisteredStorages;
|
|
}
|
|
else
|
|
{
|
|
const FString& ExistingName = IndexedCacheEntriesMetaData[StorageIndex]->GetMountName();
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("Found an existing entry named %s with at cache %d for name %s in [IndexedCacheStorage]:Storage"), *ExistingName, StorageIndex, *StorageName);
|
|
}
|
|
}
|
|
}
|
|
|
|
bHasRegisteredEntries = (RegisteredStorages > 0);
|
|
if (bHasRegisteredEntries)
|
|
{
|
|
IndexedCacheStorageMetaData = MakeUnique<FIndexedCacheStorageMetaData>(MoveTemp(IndexedCacheEntriesMetaData));
|
|
EnsureEarlyStartupCache(CacheStorageAlreadyExists);
|
|
}
|
|
}
|
|
|
|
bool FIndexedCacheStorageManager::SupportsIndexedCacheStorage()
|
|
{
|
|
return bHasRegisteredEntries && IIndexedCacheStorage::Get().SupportsIndexedCacheStorage();
|
|
}
|
|
|
|
int32 FIndexedCacheStorageManager::GetStorageIndex(const FString& StorageName)
|
|
{
|
|
int32* StorageIndex = StorageNameToStorageIndex.Find(StorageName);
|
|
return StorageIndex ? *StorageIndex : 0;
|
|
}
|
|
|
|
uint64 FIndexedCacheStorageManager::GetCacheEarlyStartupSize(int32 CacheIndex)
|
|
{
|
|
FCacheStorageMetaData* CacheStorageMetaData = GetCacheStorageMetaData(CacheIndex);
|
|
if (CacheStorageMetaData == nullptr)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("GetCacheEarlyStartupSize skip invalid cache %d"), CacheIndex);
|
|
return 0;
|
|
}
|
|
|
|
return CacheStorageMetaData->GetEarlyStartupSize();
|
|
}
|
|
|
|
void FIndexedCacheStorageManager::EnumerateCacheStorages(TArray<FString>& OutCacheStorageNames)
|
|
{
|
|
OutCacheStorageNames.Reserve(StorageNameToStorageIndex.Num());
|
|
for (const TPair<FString, int32>& StorageNameToStorageIndexIter : StorageNameToStorageIndex)
|
|
{
|
|
OutCacheStorageNames.Add(StorageNameToStorageIndexIter.Key);
|
|
}
|
|
}
|
|
|
|
bool FIndexedCacheStorageManager::CreateCacheStorage(uint64 RequestNumberOfBytes, int32 CacheIndex)
|
|
{
|
|
FCacheStorageMetaData* CacheStorageMetaData = GetCacheStorageMetaData(CacheIndex);
|
|
if (CacheStorageMetaData == nullptr)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("CreateCacheStorage skip invalid cache %d"), CacheIndex);
|
|
return false;
|
|
}
|
|
|
|
return CacheStorageMetaData->CreateCacheStorage(RequestNumberOfBytes);
|
|
}
|
|
|
|
void FIndexedCacheStorageManager::DestroyCacheStorage(int32 CacheIndex)
|
|
{
|
|
FCacheStorageMetaData* CacheStorageMetaData = GetCacheStorageMetaData(CacheIndex);
|
|
if (CacheStorageMetaData == nullptr)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("DestroyCacheStorage skip invalid cache %d"), CacheIndex);
|
|
return;
|
|
}
|
|
|
|
CacheStorageMetaData->DestroyCacheStorage();
|
|
}
|
|
|
|
uint64 FIndexedCacheStorageManager::GetCacheStorageCapacity(int32 CacheIndex)
|
|
{
|
|
FCacheStorageMetaData* CacheStorageMetaData = GetCacheStorageMetaData(CacheIndex);
|
|
if (CacheStorageMetaData == nullptr)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Warning, TEXT("GetCacheStorageCapacity skip invalid cache %d"), CacheIndex);
|
|
return 0;
|
|
}
|
|
|
|
return CacheStorageMetaData->GetCacheStorageInfo();
|
|
}
|
|
|
|
FString FIndexedCacheStorageManager::MountCacheStorage(int32 CacheIndex)
|
|
{
|
|
FCacheStorageMetaData* CacheStorageMetaData = GetCacheStorageMetaData(CacheIndex);
|
|
if (CacheStorageMetaData == nullptr)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Warning, TEXT("MountCacheStorage skip invalid cache %d"), CacheIndex);
|
|
return FString();
|
|
}
|
|
|
|
return CacheStorageMetaData->MountCacheStorage();
|
|
}
|
|
|
|
void FIndexedCacheStorageManager::UnmountCacheStorage(int32 CacheIndex)
|
|
{
|
|
FCacheStorageMetaData* CacheStorageMetaData = GetCacheStorageMetaData(CacheIndex);
|
|
if (CacheStorageMetaData == nullptr)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Warning, TEXT("UnmountCacheStorage skip invalid cache %d"), CacheIndex);
|
|
return;
|
|
}
|
|
|
|
return CacheStorageMetaData->UnmountCacheStorage();
|
|
}
|
|
|
|
FString FIndexedCacheStorageManager::GetMountName(int32 CacheIndex)
|
|
{
|
|
const FCacheStorageMetaData* CacheStorageMetaData = GetCacheStorageMetaData(CacheIndex);
|
|
if (CacheStorageMetaData == nullptr)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Warning, TEXT("GetMountName skip invalid cache %d"), CacheIndex);
|
|
return FString();
|
|
}
|
|
|
|
return CacheStorageMetaData->GetMountName();
|
|
}
|
|
|
|
FString FIndexedCacheStorageManager::GetMountPath(int32 CacheIndex)
|
|
{
|
|
const FCacheStorageMetaData* CacheStorageMetaData = GetCacheStorageMetaData(CacheIndex);
|
|
if (CacheStorageMetaData == nullptr)
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Warning, TEXT("GetMountPath skip invalid cache %d"), CacheIndex);
|
|
return FString();
|
|
}
|
|
|
|
return CacheStorageMetaData->GetMountPath();
|
|
}
|
|
|
|
FCacheStorageMetaData* FIndexedCacheStorageManager::GetCacheStorageMetaData(int32 CacheIndex)
|
|
{
|
|
// We must skip index 0 since it's mapped to GamePersistentDownloadDir()
|
|
bool bValid = CacheIndex > 0 && IndexedCacheStorageMetaData && CacheIndex < IndexedCacheStorageMetaData->IndexedCacheEntriesMetaData.Num();
|
|
if (!bValid)
|
|
{
|
|
return nullptr;
|
|
}
|
|
|
|
FCacheStorageMetaData* OutCacheStorageMetaData = IndexedCacheStorageMetaData->IndexedCacheEntriesMetaData[CacheIndex].Get();
|
|
if (OutCacheStorageMetaData && OutCacheStorageMetaData->IsValid())
|
|
{
|
|
return OutCacheStorageMetaData;
|
|
}
|
|
|
|
return nullptr;
|
|
}
|
|
|
|
#if !UE_BUILD_SHIPPING
|
|
namespace IndexedCacheStorageManager_Debug
|
|
{
|
|
static void CopyFiles(const FString& TargetDirectory);
|
|
static void ListFiles();
|
|
|
|
static FAutoConsoleCommand GCopyFiles(
|
|
TEXT("ics.CopyFiles"),
|
|
TEXT("Copies all cache files to the supplied folder"),
|
|
FConsoleCommandWithArgsDelegate::CreateLambda( [](const TArray<FString>& Args)
|
|
{
|
|
if (Args.IsEmpty())
|
|
{
|
|
UE_LOG(LogIndexedCacheStorageManager, Error, TEXT("usage: ics.CopyFiles <DestDir>"));
|
|
return;
|
|
}
|
|
|
|
CopyFiles(Args[0]);
|
|
}));
|
|
|
|
static FAutoConsoleCommand GListFiles(
|
|
TEXT("ics.ListFiles"),
|
|
TEXT("Copies all cache files to the supplied folder"),
|
|
FConsoleCommandWithArgsDelegate::CreateLambda( [](const TArray<FString>&)
|
|
{
|
|
ListFiles();
|
|
}));
|
|
|
|
static void CollectCacheIndices(TArray<int32>& CacheIndices)
|
|
{
|
|
uint32 NumIndexedCacheEntries = IIndexedCacheStorage::Get().GetNumIndexedCacheStorages();
|
|
CacheIndices.Reserve(NumIndexedCacheEntries+1);
|
|
|
|
if (FPaths::HasProjectPersistentDownloadDir())
|
|
{
|
|
CacheIndices.Add(0);
|
|
}
|
|
|
|
for (uint32 CacheIndex = 1; CacheIndex < NumIndexedCacheEntries; ++CacheIndex)
|
|
{
|
|
uint64 CacheCapacity = FIndexedCacheStorageManager::Get().GetCacheStorageCapacity(CacheIndex);
|
|
if (CacheCapacity == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
CacheIndices.Add(CacheIndex);
|
|
}
|
|
|
|
}
|
|
|
|
static void ListFiles()
|
|
{
|
|
TArray<int32> CacheIndices;
|
|
CollectCacheIndices(CacheIndices);
|
|
|
|
UE_LOG(LogConsoleResponse, Display, TEXT("Listing cache files ..."));
|
|
|
|
AsyncTask(
|
|
ENamedThreads::AnyBackgroundThreadNormalTask, [CacheIndices]() mutable
|
|
{
|
|
for (int32 CacheIndex : CacheIndices)
|
|
{
|
|
FIndexedCacheStorageScopeMount IndexedCacheStorageScopeMount(CacheIndex);
|
|
|
|
FString SourcePath = (CacheIndex == 0) ? FPaths::ProjectPersistentDownloadDir() : IndexedCacheStorageScopeMount.MountPath;
|
|
|
|
if (SourcePath.IsEmpty())
|
|
{
|
|
return;
|
|
}
|
|
|
|
IFileManager::Get().IterateDirectoryStatRecursively(
|
|
*SourcePath,
|
|
[](const TCHAR* FileOrDir, const FFileStatData& StatData)
|
|
{
|
|
if (StatData.bIsValid)
|
|
{
|
|
int64 FileSize = !StatData.bIsDirectory ? StatData.FileSize : 0;
|
|
UE_LOG(LogConsoleResponse, Display, TEXT("%10llu %s%c"), FileSize, FileOrDir, (StatData.bIsDirectory ? TEXT('/') : TEXT('\0')));
|
|
}
|
|
return true;
|
|
}
|
|
);
|
|
}
|
|
|
|
UE_LOG(LogConsoleResponse, Display, TEXT("Listing cache files done"));
|
|
|
|
}
|
|
);
|
|
}
|
|
|
|
static void CopyFiles(const FString& InTargetDirectory)
|
|
{
|
|
if (!IFileManager::Get().DirectoryExists(*InTargetDirectory))
|
|
{
|
|
IFileManager::Get().MakeDirectory(*InTargetDirectory);
|
|
}
|
|
|
|
TArray<int32> CacheIndices;
|
|
CollectCacheIndices(CacheIndices);
|
|
|
|
FString TargetDirectory(InTargetDirectory);
|
|
|
|
UE_LOG(LogConsoleResponse, Display, TEXT("Copy files to %s ..."), *TargetDirectory);
|
|
|
|
AsyncTask(
|
|
ENamedThreads::AnyBackgroundThreadNormalTask, [CacheIndices, TargetDirectory]() mutable
|
|
{
|
|
for (int32 CacheIndex : CacheIndices)
|
|
{
|
|
FIndexedCacheStorageScopeMount IndexedCacheStorageScopeMount(CacheIndex);
|
|
|
|
FString SourcePath = (CacheIndex == 0) ? FPaths::ProjectPersistentDownloadDir() : IndexedCacheStorageScopeMount.MountPath;
|
|
|
|
if (SourcePath.IsEmpty())
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (!SourcePath.EndsWith(TEXT("/")))
|
|
{
|
|
SourcePath += TEXT("/");
|
|
}
|
|
|
|
TArray<FString> SourceFiles;
|
|
TArray<FString> DestFiles;
|
|
|
|
FString TargetDirectoryCache = FString::Printf(TEXT("%s/cache_%02d"), *TargetDirectory, CacheIndex);
|
|
|
|
IFileManager::Get().IterateDirectoryStatRecursively(
|
|
*SourcePath,
|
|
[SourcePath, TargetDirectoryCache, &SourceFiles, &DestFiles](const TCHAR* FileOrDir, const FFileStatData& StatData)
|
|
{
|
|
if (StatData.bIsValid)
|
|
{
|
|
FString RelativeSourceFile(FileOrDir);
|
|
|
|
if (StatData.bIsDirectory && !RelativeSourceFile.EndsWith(TEXT("/")))
|
|
{
|
|
RelativeSourceFile += TEXT("/");
|
|
}
|
|
|
|
FPaths::MakePathRelativeTo(RelativeSourceFile, *SourcePath);
|
|
FString DestFile = TargetDirectoryCache / RelativeSourceFile;
|
|
FString DestDir = StatData.bIsDirectory ? DestFile : FPaths::GetPath(DestFile);
|
|
|
|
if (StatData.bIsDirectory)
|
|
{
|
|
IFileManager::Get().MakeDirectory(*DestDir, true);
|
|
}
|
|
else
|
|
{
|
|
SourceFiles.Add(FileOrDir);
|
|
DestFiles.Add(DestFile);
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
);
|
|
|
|
for (int32 SourceFileIndex = 0; SourceFileIndex < SourceFiles.Num(); ++SourceFileIndex)
|
|
{
|
|
UE_LOG(LogConsoleResponse, Display, TEXT("[%d/%d] %s -> %s"), SourceFileIndex+1, SourceFiles.Num(), *SourceFiles[SourceFileIndex], *DestFiles[SourceFileIndex]);
|
|
IFileManager::Get().Copy(*DestFiles[SourceFileIndex], *SourceFiles[SourceFileIndex]);
|
|
}
|
|
}
|
|
|
|
UE_LOG(LogConsoleResponse, Display, TEXT("Copy files to %s done"), *TargetDirectory);
|
|
}
|
|
);
|
|
}
|
|
|
|
} // namespace IndexedCacheStorageManager_Debug
|
|
#endif // !UE_BUILD_SHIPPING
|
|
|
|
}
|