|
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using InternalSyntax = Microsoft.CodeAnalysis.CSharp.Syntax.InternalSyntax;
namespace Microsoft.CodeAnalysis.CSharp;
/// <summary>
/// A token parser that can be used to parse tokens continuously from a source. This parser parses continuously; every call to
/// <see cref="ParseNextToken"/> will return the next token in the source text, starting from position 0. <see cref="SkipForwardTo(int)"/>
/// can be used to skip forward in the file to a specific position, and <see cref="ResetTo(Result)"/> can be used to reset the parser
/// to a previously-lexed position.
/// </summary>
/// <remarks>
/// This type is safe to double dispose, but it is not safe to use after it has been disposed. Behavior in such scenarios
/// is undefined.
/// <para />
/// This type is not thread safe.
/// </remarks>
[Experimental(RoslynExperiments.SyntaxTokenParser, UrlFormat = RoslynExperiments.SyntaxTokenParser_Url)]
public sealed class SyntaxTokenParser : IDisposable
{
private InternalSyntax.Lexer _lexer;
internal SyntaxTokenParser(InternalSyntax.Lexer lexer)
{
_lexer = lexer;
}
public void Dispose()
{
var lexer = Interlocked.CompareExchange(ref _lexer!, null, _lexer);
lexer?.Dispose();
}
/// <summary>
/// Parse the next token from the input at the current position. This will advance the internal position of the token parser to the
/// end of the returned token, including any trailing trivia.
/// </summary>
/// <remarks>
/// The returned token will have a parent of <see langword="null"/>.
/// <para />
/// Since this API does not create a <see cref="SyntaxNode"/> that owns all produced tokens,
/// the <see cref="SyntaxToken.GetLocation"/> API may yield surprising results for
/// the produced tokens and its behavior is generally unspecified.
/// </remarks>
public Result ParseNextToken()
{
var startingDirectiveStack = _lexer.Directives;
var startingPosition = _lexer.TextWindow.Position;
var token = _lexer.Lex(InternalSyntax.LexerMode.Syntax);
return new Result(new SyntaxToken(parent: null, token, startingPosition, index: 0), startingDirectiveStack);
}
/// <summary>
/// Parse the leading trivia of the next token from the input at the current position. This will advance the internal position of the
/// token parser to the end of the leading trivia of the next token. The returned result will have a token with <see cref="CSharpExtensions.Kind(SyntaxToken)"/>
/// of <see cref="SyntaxKind.None"/>, <see cref="SyntaxToken.IsMissing"/> set to <see langword="true"/>, and a parent of <see langword="null"/>. The
/// parsed trivia will be set as the <see cref="SyntaxToken.LeadingTrivia"/> of the token.
/// </summary>
public Result ParseLeadingTrivia()
{
var startingDirectiveStack = _lexer.Directives;
var startingPosition = _lexer.TextWindow.Position;
var leadingTrivia = _lexer.LexSyntaxLeadingTrivia();
var containingToken = InternalSyntax.SyntaxFactory.MissingToken(leading: leadingTrivia.Node, SyntaxKind.None, trailing: null);
return new Result(new SyntaxToken(parent: null, containingToken, startingPosition, index: 0), startingDirectiveStack);
}
/// <summary>
/// Parse syntax trivia from the current position, according to the rules of trailing syntax trivia. This will advance the internal position of the
/// token parser to the end of the trailing trivia from the current location. The returned result will have a token with <see cref="CSharpExtensions.Kind(SyntaxToken)"/>
/// of <see cref="SyntaxKind.None"/>, <see cref="SyntaxToken.IsMissing"/> set to <see langword="true"/>, and a parent of <see langword="null"/>. The
/// parsed trivia will be set as the <see cref="SyntaxToken.TrailingTrivia"/> of the token.
/// </summary>
public Result ParseTrailingTrivia()
{
var startingDirectiveStack = _lexer.Directives;
var startingPosition = _lexer.TextWindow.Position;
var trailingTrivia = _lexer.LexSyntaxTrailingTrivia();
var containingToken = InternalSyntax.SyntaxFactory.MissingToken(leading: null, SyntaxKind.None, trailing: trailingTrivia.Node);
return new Result(new SyntaxToken(parent: null, containingToken, startingPosition, index: 0), startingDirectiveStack);
}
/// <summary>
/// Skip forward in the input to the specified position. Current directive state is preserved during the skip.
/// </summary>
/// <param name="position">The absolute location in the original text to move to.</param>
/// <exception cref="ArgumentOutOfRangeException">If the given position is less than the current position of the lexer.</exception>
public void SkipForwardTo(int position)
{
if (position < _lexer.TextWindow.Position)
throw new ArgumentOutOfRangeException(nameof(position));
_lexer.TextWindow.Reset(position);
}
/// <summary>
/// Resets the token parser to an earlier position in the input. The parser is reset to the start of the token that was previously
/// parsed, before any leading trivia, with the directive state that existed at the start of the token.
/// </summary>
public void ResetTo(Result result)
{
_lexer.Reset(result.Token.Position, result.ContextStartDirectiveStack);
}
/// <summary>
/// The result of a call to <see cref="ParseNextToken"/>. This is also a context object that can be used to reset the parser to
/// before the token it represents was parsed.
/// </summary>
/// <remarks>
/// This type is not default safe. Attempts to use <code>default(Result)</code> will result in undefined behavior.
/// </remarks>
public readonly struct Result
{
/// <summary>
/// The token that was parsed.
/// </summary>
public readonly SyntaxToken Token { get; }
/// <summary>
/// If the parsed token is potentially a contextual keyword, this will return the contextual kind of the token. Otherwise, it
/// will return <see cref="SyntaxKind.None"/>.
/// </summary>
public readonly SyntaxKind ContextualKind
{
get
{
var contextualKind = Token.ContextualKind();
return contextualKind == Token.Kind() ? SyntaxKind.None : contextualKind;
}
}
internal readonly InternalSyntax.DirectiveStack ContextStartDirectiveStack;
internal Result(SyntaxToken token, InternalSyntax.DirectiveStack contextStartDirectiveStack)
{
Token = token;
ContextStartDirectiveStack = contextStartDirectiveStack;
}
}
}
|