File: Language\Legacy\DirectiveHtmlTokenizer.cs
Web Access
Project: src\src\Razor\src\Compiler\Microsoft.CodeAnalysis.Razor.Compiler\src\Microsoft.CodeAnalysis.Razor.Compiler.csproj (Microsoft.CodeAnalysis.Razor.Compiler)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
 
#nullable disable
 
using System.Diagnostics;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
 
namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
internal class DirectiveHtmlTokenizer(SeekableTextReader source) : HtmlTokenizer(source)
{
    private bool _visitedFirstTokenStart;
    private SourceLocation _firstTokenVisitLocation = SourceLocation.Undefined;
 
    protected override StateResult Dispatch()
    {
        var location = CurrentLocation;
        var result = base.Dispatch();
        if (result.Result != null && IsValidTokenType(result.Result.Kind))
        {
            _visitedFirstTokenStart = true;
            _firstTokenVisitLocation = location;
        }
 
        return result;
    }
 
    public override SyntaxToken NextToken()
    {
        // Post-Condition: Buffer should be empty at the start of Next()
        Debug.Assert(Buffer.Length == 0);
        StartToken();
 
        if (EndOfFile || (_visitedFirstTokenStart && _firstTokenVisitLocation != CurrentLocation))
        {
            // We also need to make sure we are currently past the position where we found the first token.
            // If the position is equal, that means the parser put the token back for later parsing.
            return null;
        }
 
        var token = Turn();
 
        // Post-Condition: Buffer should be empty at the end of Next()
        Debug.Assert(Buffer.Length == 0);
 
        return token;
    }
 
    private bool IsValidTokenType(SyntaxKind kind)
    {
        return kind != SyntaxKind.Whitespace &&
            kind != SyntaxKind.NewLine &&
            kind != SyntaxKind.RazorCommentLiteral &&
            kind != SyntaxKind.RazorCommentStar &&
            kind != SyntaxKind.RazorCommentTransition &&
            kind != SyntaxKind.Transition;
    }
}