// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.Threading; using EpicGames.Core; using EpicGames.UHT.Utils; namespace EpicGames.UHT.Tokenizer { /// /// Token reader to replay previously recorded token stream /// public sealed class UhtTokenReplayReader : IUhtTokenReader, IUhtMessageLineNumber { const int MaxSavedStates = 2; private struct SavedState { public int TokenIndex { get; set; } public bool HasToken { get; set; } } private IUhtMessageSite _messageSite; private ReadOnlyMemory _tokens; private ReadOnlyMemory _data; private int _currentTokenIndex = -1; private bool _hasToken = false; private UhtToken _currentToken = new(); private readonly SavedState[] _savedStates = new SavedState[MaxSavedStates]; private int _savedStateCount = 0; private UhtTokenType _endTokenType = UhtTokenType.EndOfFile; /// /// Construct new token reader /// /// Message site for generating errors /// Complete data for the token (i.e. original source) /// Tokens to replay /// Token type to return when end of tokens reached public UhtTokenReplayReader(IUhtMessageSite messageSite, ReadOnlyMemory data, ReadOnlyMemory tokens, UhtTokenType endTokenType) { _messageSite = messageSite; _tokens = tokens; _data = data; _endTokenType = endTokenType; _currentToken = new UhtToken(endTokenType); } /// /// Construct token replay reader intended for caching. Use Reset method to prepare it for use /// public UhtTokenReplayReader() { _messageSite = new UhtEmptyMessageSite(); _tokens = Array.Empty().AsMemory(); _data = Array.Empty().AsMemory(); } /// /// Reset a cached replay reader for replaying a new stream of tokens /// /// Message site for generating errors /// Complete data for the token (i.e. original source) /// Tokens to replay /// Token type to return when end of tokens reached /// The replay reader public UhtTokenReplayReader Reset(IUhtMessageSite messageSite, ReadOnlyMemory data, ReadOnlyMemory tokens, UhtTokenType endTokenType) { _messageSite = messageSite; _tokens = tokens; _data = data; _currentTokenIndex = -1; _hasToken = false; _currentToken = new UhtToken(endTokenType); _savedStateCount = 0; _endTokenType = endTokenType; return this; } #region IUHTMessageSite Implementation IUhtMessageSession IUhtMessageSite.MessageSession => _messageSite.MessageSession; IUhtMessageSource? IUhtMessageSite.MessageSource => _messageSite.MessageSource; IUhtMessageLineNumber? IUhtMessageSite.MessageLineNumber => this; #endregion #region ITokenReader Implementation /// public bool IsEOF { get { if (!_hasToken) { GetTokenInternal(); } return _currentTokenIndex == _tokens.Span.Length; } } /// public int InputPos { get { if (!_hasToken) { GetTokenInternal(); } return _currentToken.InputStartPos; } } /// public int InputLine { get { if (!_hasToken) { GetTokenInternal(); } return _currentToken.InputLine; } set => throw new NotImplementedException(); } /// public IUhtTokenPreprocessor? TokenPreprocessor { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } /// public int LookAheadEnableCount { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } /// public ReadOnlySpan Comments => throw new NotImplementedException(); /// public void ClearComments() { throw new NotImplementedException(); } /// public void ConsumeToken() { _hasToken = false; } /// public void DisableComments() { throw new NotImplementedException(); } /// public void EnableComments() { throw new NotImplementedException(); } /// public void CommitPendingComments() { throw new NotImplementedException(); } /// public UhtToken GetLine() { throw new NotImplementedException(); } /// public StringView GetRawString(char terminator, UhtRawStringOptions options) { throw new NotImplementedException(); } /// public StringView GetStringView(int startPos, int count) { return new StringView(_data, startPos, count); } /// public UhtToken GetToken() { UhtToken token = PeekToken(); ConsumeToken(); return token; } /// public int GetReplayTokenIndex() { return _hasToken ? _currentTokenIndex : _currentTokenIndex + 1; } /// public bool IsFirstTokenInLine(ref UhtToken token) { throw new NotImplementedException(); } /// public ref UhtToken PeekToken() { if (!_hasToken) { GetTokenInternal(); } return ref _currentToken; } /// public void SkipWhitespaceAndComments() { throw new NotImplementedException(); } /// public void SaveState() { if (_savedStateCount == MaxSavedStates) { throw new UhtIceException("Token reader saved states full"); } _savedStates[_savedStateCount] = new SavedState { TokenIndex = _currentTokenIndex, HasToken = _hasToken }; ++_savedStateCount; } /// public void RestoreState() { if (_savedStateCount == 0) { throw new UhtIceException("Attempt to restore a state when none have been saved"); } --_savedStateCount; _currentTokenIndex = _savedStates[_savedStateCount].TokenIndex; _hasToken = _savedStates[_savedStateCount].HasToken; if (_hasToken) { _currentToken = _tokens.Span[_currentTokenIndex]; } } /// public void AbandonState() { if (_savedStateCount == 0) { throw new UhtIceException("Attempt to abandon a state when none have been saved"); } --_savedStateCount; } /// public void EnableRecording() { throw new NotImplementedException(); } /// public void DisableRecording() { throw new NotImplementedException(); } /// public void RecordToken(ref UhtToken token) { throw new NotImplementedException(); } /// public List RecordedTokens => throw new NotImplementedException(); #endregion #region IUHTMessageLineNumber implementation int IUhtMessageLineNumber.MessageLineNumber { get { if (_tokens.Length == 0) { return -1; } if (_currentTokenIndex < 0) { return _tokens.Span[0].InputLine; } if (_currentTokenIndex < _tokens.Length) { return _tokens.Span[_currentTokenIndex].InputLine; } return _tokens.Span[_tokens.Length - 1].InputLine; } } #endregion private ref UhtToken GetTokenInternal() { if (_currentTokenIndex < _tokens.Length) { ++_currentTokenIndex; } if (_currentTokenIndex < _tokens.Length) { _currentToken = _tokens.Span[_currentTokenIndex]; } else if (_tokens.Length == 0) { _currentToken = new UhtToken(); } else { UhtToken lastToken = _tokens.Span[_tokens.Length - 1]; int endPos = lastToken.InputEndPos; _currentToken = new UhtToken(_endTokenType, endPos, lastToken.InputLine, endPos, lastToken.InputLine, new StringView()); } _hasToken = true; return ref _currentToken; } } /// /// Borrow a thread specific token replay reader /// public readonly struct UhtTokenReplayReaderBorrower : IDisposable { private static readonly ThreadLocal> s_tlsStack = new(() => new Stack()); private readonly UhtTokenReplayReader _reader; /// /// The borrowed reader /// public readonly IUhtTokenReader Reader => _reader; /// /// Borrow a thread specific replay reader /// /// Location for messages /// Source being parsed /// Tokens to be replayed /// Token type when end of tokens reached /// public UhtTokenReplayReaderBorrower(IUhtMessageSite messageSite, ReadOnlyMemory data, ReadOnlyMemory tokens, UhtTokenType endTokenType) { Stack stack = s_tlsStack.Value!; _reader = stack.Count > 0 ? _reader = stack.Pop() : new(); _reader.Reset(messageSite, data, tokens, endTokenType); } /// public readonly void Dispose() { s_tlsStack.Value!.Push(_reader); } } }