// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.Threading;
using EpicGames.Core;
using EpicGames.UHT.Utils;
namespace EpicGames.UHT.Tokenizer
{
///
/// Token reader to replay previously recorded token stream
///
public sealed class UhtTokenReplayReader : IUhtTokenReader, IUhtMessageLineNumber
{
const int MaxSavedStates = 2;
private struct SavedState
{
public int TokenIndex { get; set; }
public bool HasToken { get; set; }
}
private IUhtMessageSite _messageSite;
private ReadOnlyMemory _tokens;
private ReadOnlyMemory _data;
private int _currentTokenIndex = -1;
private bool _hasToken = false;
private UhtToken _currentToken = new();
private readonly SavedState[] _savedStates = new SavedState[MaxSavedStates];
private int _savedStateCount = 0;
private UhtTokenType _endTokenType = UhtTokenType.EndOfFile;
///
/// Construct new token reader
///
/// Message site for generating errors
/// Complete data for the token (i.e. original source)
/// Tokens to replay
/// Token type to return when end of tokens reached
public UhtTokenReplayReader(IUhtMessageSite messageSite, ReadOnlyMemory data, ReadOnlyMemory tokens, UhtTokenType endTokenType)
{
_messageSite = messageSite;
_tokens = tokens;
_data = data;
_endTokenType = endTokenType;
_currentToken = new UhtToken(endTokenType);
}
///
/// Construct token replay reader intended for caching. Use Reset method to prepare it for use
///
public UhtTokenReplayReader()
{
_messageSite = new UhtEmptyMessageSite();
_tokens = Array.Empty().AsMemory();
_data = Array.Empty().AsMemory();
}
///
/// Reset a cached replay reader for replaying a new stream of tokens
///
/// Message site for generating errors
/// Complete data for the token (i.e. original source)
/// Tokens to replay
/// Token type to return when end of tokens reached
/// The replay reader
public UhtTokenReplayReader Reset(IUhtMessageSite messageSite, ReadOnlyMemory data, ReadOnlyMemory tokens, UhtTokenType endTokenType)
{
_messageSite = messageSite;
_tokens = tokens;
_data = data;
_currentTokenIndex = -1;
_hasToken = false;
_currentToken = new UhtToken(endTokenType);
_savedStateCount = 0;
_endTokenType = endTokenType;
return this;
}
#region IUHTMessageSite Implementation
IUhtMessageSession IUhtMessageSite.MessageSession => _messageSite.MessageSession;
IUhtMessageSource? IUhtMessageSite.MessageSource => _messageSite.MessageSource;
IUhtMessageLineNumber? IUhtMessageSite.MessageLineNumber => this;
#endregion
#region ITokenReader Implementation
///
public bool IsEOF
{
get
{
if (!_hasToken)
{
GetTokenInternal();
}
return _currentTokenIndex == _tokens.Span.Length;
}
}
///
public int InputPos
{
get
{
if (!_hasToken)
{
GetTokenInternal();
}
return _currentToken.InputStartPos;
}
}
///
public int InputLine
{
get
{
if (!_hasToken)
{
GetTokenInternal();
}
return _currentToken.InputLine;
}
set => throw new NotImplementedException();
}
///
public IUhtTokenPreprocessor? TokenPreprocessor { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
///
public int LookAheadEnableCount { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
///
public ReadOnlySpan Comments => throw new NotImplementedException();
///
public void ClearComments()
{
throw new NotImplementedException();
}
///
public void ConsumeToken()
{
_hasToken = false;
}
///
public void DisableComments()
{
throw new NotImplementedException();
}
///
public void EnableComments()
{
throw new NotImplementedException();
}
///
public void CommitPendingComments()
{
throw new NotImplementedException();
}
///
public UhtToken GetLine()
{
throw new NotImplementedException();
}
///
public StringView GetRawString(char terminator, UhtRawStringOptions options)
{
throw new NotImplementedException();
}
///
public StringView GetStringView(int startPos, int count)
{
return new StringView(_data, startPos, count);
}
///
public UhtToken GetToken()
{
UhtToken token = PeekToken();
ConsumeToken();
return token;
}
///
public int GetReplayTokenIndex()
{
return _hasToken ? _currentTokenIndex : _currentTokenIndex + 1;
}
///
public bool IsFirstTokenInLine(ref UhtToken token)
{
throw new NotImplementedException();
}
///
public ref UhtToken PeekToken()
{
if (!_hasToken)
{
GetTokenInternal();
}
return ref _currentToken;
}
///
public void SkipWhitespaceAndComments()
{
throw new NotImplementedException();
}
///
public void SaveState()
{
if (_savedStateCount == MaxSavedStates)
{
throw new UhtIceException("Token reader saved states full");
}
_savedStates[_savedStateCount] = new SavedState { TokenIndex = _currentTokenIndex, HasToken = _hasToken };
++_savedStateCount;
}
///
public void RestoreState()
{
if (_savedStateCount == 0)
{
throw new UhtIceException("Attempt to restore a state when none have been saved");
}
--_savedStateCount;
_currentTokenIndex = _savedStates[_savedStateCount].TokenIndex;
_hasToken = _savedStates[_savedStateCount].HasToken;
if (_hasToken)
{
_currentToken = _tokens.Span[_currentTokenIndex];
}
}
///
public void AbandonState()
{
if (_savedStateCount == 0)
{
throw new UhtIceException("Attempt to abandon a state when none have been saved");
}
--_savedStateCount;
}
///
public void EnableRecording()
{
throw new NotImplementedException();
}
///
public void DisableRecording()
{
throw new NotImplementedException();
}
///
public void RecordToken(ref UhtToken token)
{
throw new NotImplementedException();
}
///
public List RecordedTokens => throw new NotImplementedException();
#endregion
#region IUHTMessageLineNumber implementation
int IUhtMessageLineNumber.MessageLineNumber
{
get
{
if (_tokens.Length == 0)
{
return -1;
}
if (_currentTokenIndex < 0)
{
return _tokens.Span[0].InputLine;
}
if (_currentTokenIndex < _tokens.Length)
{
return _tokens.Span[_currentTokenIndex].InputLine;
}
return _tokens.Span[_tokens.Length - 1].InputLine;
}
}
#endregion
private ref UhtToken GetTokenInternal()
{
if (_currentTokenIndex < _tokens.Length)
{
++_currentTokenIndex;
}
if (_currentTokenIndex < _tokens.Length)
{
_currentToken = _tokens.Span[_currentTokenIndex];
}
else if (_tokens.Length == 0)
{
_currentToken = new UhtToken();
}
else
{
UhtToken lastToken = _tokens.Span[_tokens.Length - 1];
int endPos = lastToken.InputEndPos;
_currentToken = new UhtToken(_endTokenType, endPos, lastToken.InputLine, endPos, lastToken.InputLine, new StringView());
}
_hasToken = true;
return ref _currentToken;
}
}
///
/// Borrow a thread specific token replay reader
///
public readonly struct UhtTokenReplayReaderBorrower : IDisposable
{
private static readonly ThreadLocal> s_tlsStack = new(() => new Stack());
private readonly UhtTokenReplayReader _reader;
///
/// The borrowed reader
///
public readonly IUhtTokenReader Reader => _reader;
///
/// Borrow a thread specific replay reader
///
/// Location for messages
/// Source being parsed
/// Tokens to be replayed
/// Token type when end of tokens reached
///
public UhtTokenReplayReaderBorrower(IUhtMessageSite messageSite, ReadOnlyMemory data, ReadOnlyMemory tokens, UhtTokenType endTokenType)
{
Stack stack = s_tlsStack.Value!;
_reader = stack.Count > 0 ? _reader = stack.Pop() : new();
_reader.Reset(messageSite, data, tokens, endTokenType);
}
///
public readonly void Dispose()
{
s_tlsStack.Value!.Push(_reader);
}
}
}