From 00c13afa7f7637466e455bdc0e4d78af7434c82c Mon Sep 17 00:00:00 2001
From: Ajay Bhargav Baaskaran
Date: Tue, 21 Aug 2018 17:33:07 -0700
Subject: [PATCH 01/23] Razor parser rewrite - Rewrite CSharp parser - Basic
rewrite of HTML parser - Define and generate syntax nodes and boilerplate -
Rewrite ClassifiedSpan generation logic - Update parser test infrastructure
---
.../ClassifiedSpanVisitor.cs | 279 ++
.../DefaultDirectiveSyntaxTreePass.cs | 48 +-
.../DefaultRazorSyntaxTree.cs | 11 +-
.../Legacy/CSharpCodeParser.cs | 77 +-
.../Legacy/CSharpLanguageCharacteristics.cs | 2 +-
.../Legacy/CSharpParser.cs | 2123 +++++++++++++
.../Legacy/HtmlLanguageCharacteristics.cs | 2 +-
.../Legacy/HtmlMarkupParser.cs | 49 +-
.../Legacy/HtmlParser.cs | 1741 +++++++++++
.../Legacy/KnownTokenType.cs | 2 +-
.../Legacy/LanguageCharacteristics.cs | 4 +-
.../Legacy/ParserBase.cs | 4 +-
.../Legacy/ParserContext.cs | 4 +
.../Legacy/RazorParser.cs | 2 +-
.../Legacy/RazorSyntaxTreeExtensions.cs | 14 +
.../Legacy/SpanContext.cs | 46 +
.../Legacy/SyntaxConstants.cs | 1 +
.../Legacy/TokenizerBackedParser.cs | 411 ++-
.../LegacyRazorSyntaxTree.cs | 31 +
.../RazorSyntaxTree.cs | 32 +
.../Syntax.xml.Internal.Generated.cs | 2651 ++++++++++++++---
.../Generated/Syntax.xml.Main.Generated.cs | 676 ++++-
.../Generated/Syntax.xml.Syntax.Generated.cs | 1560 +++++++++-
.../Syntax/GreenNode.cs | 20 +-
.../Syntax/GreenNodeExtensions.cs | 4 +-
.../Syntax/InternalSyntax/SyntaxFactory.cs | 5 +
.../Syntax/InternalSyntax/SyntaxListOfT.cs | 102 +-
.../Syntax/InternalSyntax/SyntaxListPool.cs | 100 +
.../Syntax/InternalSyntax/SyntaxRewriter.cs | 56 +
.../Syntax/InternalSyntax/SyntaxToken.cs | 57 +-
.../Syntax/Syntax.xml | 104 +-
.../Syntax/SyntaxAnnotation.cs | 4 +-
.../Syntax/SyntaxKind.cs | 21 +-
.../Syntax/SyntaxNode.cs | 4 +
.../Syntax/SyntaxNodeExtensions.cs | 75 +-
.../Syntax/SyntaxRewriter.cs | 148 +
.../Syntax/SyntaxToken.cs | 17 +-
.../Legacy/CodeParserTestBase.cs | 2 +-
.../Legacy/HtmlMarkupParserTests.cs | 2 +-
.../Legacy/TokenizerLookaheadTest.cs | 2 +-
.../ClassifiedSpanSerializer.cs | 31 +-
.../ClassifiedSpan/ClassifiedSpanVerifier.cs | 32 +-
.../ClassifiedSpan/ClassifiedSpanWriter.cs | 66 +-
.../Language/Legacy/ParserTestBase.cs | 18 +-
.../Legacy/SyntaxNodeParserTestBase.cs | 164 +
.../Language/Legacy/SyntaxNodeSerializer.cs | 61 +
.../Language/Legacy/SyntaxNodeVerifier.cs | 296 ++
.../Language/Legacy/SyntaxNodeWalker.cs | 37 +
.../Language/Legacy/SyntaxNodeWriter.cs | 126 +
.../TagHelperSpan/TagHelperSpanSerializer.cs | 5 +
.../Language/SyntaxTreeVerifier.cs | 59 +-
51 files changed, 10588 insertions(+), 800 deletions(-)
create mode 100644 src/Microsoft.AspNetCore.Razor.Language/ClassifiedSpanVisitor.cs
create mode 100644 src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpParser.cs
create mode 100644 src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlParser.cs
create mode 100644 src/Microsoft.AspNetCore.Razor.Language/Legacy/SpanContext.cs
create mode 100644 src/Microsoft.AspNetCore.Razor.Language/LegacyRazorSyntaxTree.cs
create mode 100644 src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxListPool.cs
create mode 100644 src/Microsoft.AspNetCore.Razor.Language/Syntax/InternalSyntax/SyntaxRewriter.cs
create mode 100644 src/Microsoft.AspNetCore.Razor.Language/Syntax/SyntaxRewriter.cs
create mode 100644 test/Microsoft.AspNetCore.Razor.Test.Common/Language/Legacy/SyntaxNodeParserTestBase.cs
create mode 100644 test/Microsoft.AspNetCore.Razor.Test.Common/Language/Legacy/SyntaxNodeSerializer.cs
create mode 100644 test/Microsoft.AspNetCore.Razor.Test.Common/Language/Legacy/SyntaxNodeVerifier.cs
create mode 100644 test/Microsoft.AspNetCore.Razor.Test.Common/Language/Legacy/SyntaxNodeWalker.cs
create mode 100644 test/Microsoft.AspNetCore.Razor.Test.Common/Language/Legacy/SyntaxNodeWriter.cs
diff --git a/src/Microsoft.AspNetCore.Razor.Language/ClassifiedSpanVisitor.cs b/src/Microsoft.AspNetCore.Razor.Language/ClassifiedSpanVisitor.cs
new file mode 100644
index 000000000..463eae1ab
--- /dev/null
+++ b/src/Microsoft.AspNetCore.Razor.Language/ClassifiedSpanVisitor.cs
@@ -0,0 +1,279 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using Microsoft.AspNetCore.Razor.Language.Legacy;
+using Microsoft.AspNetCore.Razor.Language.Syntax;
+
+namespace Microsoft.AspNetCore.Razor.Language
+{
+ internal class ClassifiedSpanVisitor : SyntaxRewriter
+ {
+ private RazorSourceDocument _source;
+ private List _spans;
+ private BlockKindInternal _currentBlockKind;
+ private SyntaxNode _currentBlock;
+
+ public ClassifiedSpanVisitor(RazorSourceDocument source)
+ {
+ _source = source;
+ _spans = new List();
+ _currentBlockKind = BlockKindInternal.Markup;
+ }
+
+ public IReadOnlyList ClassifiedSpans => _spans;
+
+ public override SyntaxNode VisitRazorCommentBlock(RazorCommentBlockSyntax node)
+ {
+ return WriteBlock(node, BlockKindInternal.Comment, base.VisitRazorCommentBlock);
+ }
+
+ public override SyntaxNode VisitCSharpCodeBlock(CSharpCodeBlockSyntax node)
+ {
+ if (node.Parent is CSharpStatementBodySyntax ||
+ node.Parent is CSharpExpressionBodySyntax ||
+ node.Parent is CSharpImplicitExpressionBodySyntax ||
+ node.Parent is CSharpDirectiveBodySyntax)
+ {
+ return base.VisitCSharpCodeBlock(node);
+ }
+
+ return WriteBlock(node, BlockKindInternal.Statement, base.VisitCSharpCodeBlock);
+ }
+
+ public override SyntaxNode VisitCSharpStatement(CSharpStatement node)
+ {
+ return WriteBlock(node, BlockKindInternal.Statement, base.VisitCSharpStatement);
+ }
+
+ public override SyntaxNode VisitCSharpExpression(CSharpExpression node)
+ {
+ return WriteBlock(node, BlockKindInternal.Expression, base.VisitCSharpExpression);
+ }
+
+ public override SyntaxNode VisitCSharpImplicitExpression(CSharpImplicitExpression node)
+ {
+ return WriteBlock(node, BlockKindInternal.Expression, base.VisitCSharpImplicitExpression);
+ }
+
+ public override SyntaxNode VisitCSharpDirective(CSharpDirectiveSyntax node)
+ {
+ return WriteBlock(node, BlockKindInternal.Directive, base.VisitCSharpDirective);
+ }
+
+ public override SyntaxNode VisitCSharpTemplateBlock(CSharpTemplateBlockSyntax node)
+ {
+ return WriteBlock(node, BlockKindInternal.Template, base.VisitCSharpTemplateBlock);
+ }
+
+ public override SyntaxNode VisitHtmlMarkupBlock(HtmlMarkupBlockSyntax node)
+ {
+ return WriteBlock(node, BlockKindInternal.Markup, base.VisitHtmlMarkupBlock);
+ }
+
+ public override SyntaxNode VisitHtmlTagBlock(HtmlTagBlockSyntax node)
+ {
+ return WriteBlock(node, BlockKindInternal.Tag, base.VisitHtmlTagBlock);
+ }
+
+ public override SyntaxNode VisitHtmlAttributeBlock(HtmlAttributeBlockSyntax node)
+ {
+ return WriteBlock(node, BlockKindInternal.Markup, n =>
+ {
+ var equalsSyntax = SyntaxFactory.HtmlTextLiteral(new SyntaxList(node.EqualsToken));
+ var mergedAttributePrefix = MergeTextLiteralSpans(node.NamePrefix, node.Name, node.NameSuffix, equalsSyntax, node.ValuePrefix);
+ Visit(mergedAttributePrefix);
+ Visit(node.Value);
+ Visit(node.ValueSuffix);
+
+ return n;
+ });
+ }
+
+ public override SyntaxNode VisitHtmlMinimizedAttributeBlock(HtmlMinimizedAttributeBlockSyntax node)
+ {
+ return WriteBlock(node, BlockKindInternal.Markup, n =>
+ {
+ var mergedAttributePrefix = MergeTextLiteralSpans(node.NamePrefix, node.Name);
+ Visit(mergedAttributePrefix);
+
+ return n;
+ });
+ }
+
+ public override SyntaxNode VisitHtmlCommentBlock(HtmlCommentBlockSyntax node)
+ {
+ return WriteBlock(node, BlockKindInternal.HtmlComment, base.VisitHtmlCommentBlock);
+ }
+
+ public override SyntaxNode VisitHtmlDynamicAttributeValue(HtmlDynamicAttributeValueSyntax node)
+ {
+ return WriteBlock(node, BlockKindInternal.Markup, base.VisitHtmlDynamicAttributeValue);
+ }
+
+ public override SyntaxNode VisitRazorMetaCode(RazorMetaCodeSyntax node)
+ {
+ WriteSpan(node, SpanKindInternal.MetaCode);
+ return base.VisitRazorMetaCode(node);
+ }
+
+ public override SyntaxNode VisitCSharpTransition(CSharpTransitionSyntax node)
+ {
+ WriteSpan(node, SpanKindInternal.Transition);
+ return base.VisitCSharpTransition(node);
+ }
+
+ public override SyntaxNode VisitHtmlTransition(HtmlTransitionSyntax node)
+ {
+ WriteSpan(node, SpanKindInternal.Transition);
+ return base.VisitHtmlTransition(node);
+ }
+
+ public override SyntaxNode VisitCSharpStatementLiteral(CSharpStatementLiteralSyntax node)
+ {
+ WriteSpan(node, SpanKindInternal.Code);
+ return base.VisitCSharpStatementLiteral(node);
+ }
+
+ public override SyntaxNode VisitCSharpExpressionLiteral(CSharpExpressionLiteralSyntax node)
+ {
+ WriteSpan(node, SpanKindInternal.Code);
+ return base.VisitCSharpExpressionLiteral(node);
+ }
+
+ public override SyntaxNode VisitCSharpHiddenLiteral(CSharpHiddenLiteralSyntax node)
+ {
+ WriteSpan(node, SpanKindInternal.Code);
+ return base.VisitCSharpHiddenLiteral(node);
+ }
+
+ public override SyntaxNode VisitCSharpNoneLiteral(CSharpNoneLiteralSyntax node)
+ {
+ WriteSpan(node, SpanKindInternal.None);
+ return base.VisitCSharpNoneLiteral(node);
+ }
+
+ public override SyntaxNode VisitHtmlLiteralAttributeValue(HtmlLiteralAttributeValueSyntax node)
+ {
+ WriteSpan(node, SpanKindInternal.Markup);
+ return base.VisitHtmlLiteralAttributeValue(node);
+ }
+
+ public override SyntaxNode VisitHtmlTextLiteral(HtmlTextLiteralSyntax node)
+ {
+ if (node.Parent is HtmlLiteralAttributeValueSyntax)
+ {
+ return base.VisitHtmlTextLiteral(node);
+ }
+
+ WriteSpan(node, SpanKindInternal.Markup);
+ return base.VisitHtmlTextLiteral(node);
+ }
+
+ private SyntaxNode WriteBlock(TNode node, BlockKindInternal kind, Func handler) where TNode : SyntaxNode
+ {
+ var previousBlock = _currentBlock;
+ var previousKind = _currentBlockKind;
+
+ _currentBlock = node;
+ _currentBlockKind = kind;
+
+ var result = handler(node);
+
+ _currentBlock = previousBlock;
+ _currentBlockKind = previousKind;
+
+ return result;
+ }
+
+ private void WriteSpan(SyntaxNode node, SpanKindInternal kind)
+ {
+ if (node.IsMissing)
+ {
+ return;
+ }
+
+ var spanSource = GetSourceSpanForNode(node);
+ var blockSource = GetSourceSpanForNode(_currentBlock);
+ var acceptedCharacters = AcceptedCharactersInternal.Any;
+ var annotation = node.GetAnnotationValue(SyntaxConstants.SpanContextKind);
+ if (annotation is SpanContext context)
+ {
+ acceptedCharacters = context.EditHandler.AcceptedCharacters;
+ }
+
+ var span = new ClassifiedSpanInternal(spanSource, blockSource, kind, _currentBlockKind, acceptedCharacters);
+ _spans.Add(span);
+ }
+
+ private HtmlTextLiteralSyntax MergeTextLiteralSpans(params HtmlTextLiteralSyntax[] literalSyntaxes)
+ {
+ if (literalSyntaxes == null || literalSyntaxes.Length == 0)
+ {
+ return null;
+ }
+
+ SyntaxNode parent = null;
+ var position = 0;
+ var seenFirstLiteral = false;
+ var builder = Syntax.InternalSyntax.SyntaxListBuilder.Create();
+
+ foreach (var syntax in literalSyntaxes)
+ {
+ if (syntax == null)
+ {
+ continue;
+ }
+ else if (!seenFirstLiteral)
+ {
+ // Set the parent and position of the merged literal to the value of the first non-null literal.
+ parent = syntax.Parent;
+ position = syntax.Position;
+ seenFirstLiteral = true;
+ }
+
+ foreach (var token in syntax.TextTokens)
+ {
+ builder.Add(token.Green);
+ }
+ }
+
+ var mergedLiteralSyntax = Syntax.InternalSyntax.SyntaxFactory.HtmlTextLiteral(
+ builder.ToList());
+
+ return (HtmlTextLiteralSyntax)mergedLiteralSyntax.CreateRed(parent, position);
+ }
+
+ private SourceSpan GetSourceSpanForNode(SyntaxNode node)
+ {
+ try
+ {
+ if (_source.Length == 0)
+ {
+ // Just a marker symbol
+ return new SourceSpan(_source.FilePath, 0, 0, 0, node.FullWidth);
+ }
+ if (node.Position >= _source.Length)
+ {
+ // E.g. Marker symbol at the end of the document
+ var lastLocation = _source.Lines.GetLocation(_source.Length - 1);
+ return new SourceSpan(
+ lastLocation.FilePath,
+ lastLocation.AbsoluteIndex + 1,
+ lastLocation.LineIndex,
+ lastLocation.CharacterIndex + 1,
+ node.FullWidth);
+ }
+
+ return node.GetSourceSpan(_source);
+ }
+ catch (IndexOutOfRangeException)
+ {
+ Debug.Assert(false, "Node position should stay within document length.");
+ return new SourceSpan(_source.FilePath, node.Position, 0, 0, node.FullWidth);
+ }
+ }
+ }
+}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/DefaultDirectiveSyntaxTreePass.cs b/src/Microsoft.AspNetCore.Razor.Language/DefaultDirectiveSyntaxTreePass.cs
index 54c733e0f..17b6b9b9c 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/DefaultDirectiveSyntaxTreePass.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/DefaultDirectiveSyntaxTreePass.cs
@@ -5,6 +5,7 @@
using System.Linq;
using Microsoft.AspNetCore.Razor.Language.Extensions;
using Microsoft.AspNetCore.Razor.Language.Legacy;
+using Microsoft.AspNetCore.Razor.Language.Syntax;
namespace Microsoft.AspNetCore.Razor.Language
{
@@ -24,13 +25,52 @@ public RazorSyntaxTree Execute(RazorCodeDocument codeDocument, RazorSyntaxTree s
throw new ArgumentNullException(nameof(syntaxTree));
}
- var sectionVerifier = new NestedSectionVerifier();
- sectionVerifier.Verify(syntaxTree);
+ if (syntaxTree is LegacyRazorSyntaxTree)
+ {
+ var legacySectionVerifier = new LegacyNestedSectionVerifier();
+ legacySectionVerifier.Verify(syntaxTree);
+ return syntaxTree;
+ }
+
+ var sectionVerifier = new NestedSectionVerifier(syntaxTree);
+ return sectionVerifier.Verify();
+ }
+
+ private class NestedSectionVerifier : SyntaxRewriter
+ {
+ private int _nestedLevel;
+ private RazorSyntaxTree _syntaxTree;
+
+ public NestedSectionVerifier(RazorSyntaxTree syntaxTree)
+ {
+ _syntaxTree = syntaxTree;
+ }
+
+ public RazorSyntaxTree Verify()
+ {
+ var root = Visit(_syntaxTree.NewRoot);
+ var rewrittenTree = new DefaultRazorSyntaxTree(root, _syntaxTree.Source, _syntaxTree.Diagnostics, _syntaxTree.Options);
+ return rewrittenTree;
+ }
- return syntaxTree;
+ public override SyntaxNode VisitCSharpDirective(CSharpDirectiveSyntax node)
+ {
+ if (_nestedLevel > 0)
+ {
+ var directiveStart = node.Transition.GetSourceLocation(_syntaxTree.Source);
+ var errorLength = /* @ */ 1 + SectionDirective.Directive.Directive.Length;
+ var error = RazorDiagnosticFactory.CreateParsing_SectionsCannotBeNested(new SourceSpan(directiveStart, errorLength));
+ node = node.AppendDiagnostic(error);
+ }
+ _nestedLevel++;
+ var result = base.VisitCSharpDirective(node);
+ _nestedLevel--;
+
+ return result;
+ }
}
- private class NestedSectionVerifier : ParserVisitor
+ private class LegacyNestedSectionVerifier : ParserVisitor
{
private int _nestedLevel;
diff --git a/src/Microsoft.AspNetCore.Razor.Language/DefaultRazorSyntaxTree.cs b/src/Microsoft.AspNetCore.Razor.Language/DefaultRazorSyntaxTree.cs
index 4b6d43fe0..4683579ca 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/DefaultRazorSyntaxTree.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/DefaultRazorSyntaxTree.cs
@@ -1,20 +1,22 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+using System;
using System.Collections.Generic;
using Microsoft.AspNetCore.Razor.Language.Legacy;
+using Microsoft.AspNetCore.Razor.Language.Syntax;
namespace Microsoft.AspNetCore.Razor.Language
{
internal class DefaultRazorSyntaxTree : RazorSyntaxTree
{
public DefaultRazorSyntaxTree(
- Block root,
+ SyntaxNode root,
RazorSourceDocument source,
IReadOnlyList diagnostics,
RazorParserOptions options)
{
- Root = root;
+ NewRoot = root;
Source = source;
Diagnostics = diagnostics;
Options = options;
@@ -24,8 +26,11 @@ public DefaultRazorSyntaxTree(
public override RazorParserOptions Options { get; }
- internal override Block Root { get; }
+ internal override SyntaxNode NewRoot { get; }
public override RazorSourceDocument Source { get; }
+
+ // Temporary
+ internal override Block Root => throw new NotImplementedException();
}
}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs
index 126970945..23fe7b0fe 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/CSharpCodeParser.cs
@@ -9,7 +9,7 @@
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class CSharpCodeParser : TokenizerBackedParser
+ internal partial class CSharpCodeParser : TokenizerBackedParser
{
private static HashSet InvalidNonWhitespaceNameCharacters = new HashSet(new[]
{
@@ -95,6 +95,10 @@ public CSharpCodeParser(IEnumerable directives, ParserConte
SetUpKeywords();
SetupDirectives(directives);
SetUpExpressions();
+
+ SetupKeywordParsers();
+ SetupExpressionParsers();
+ SetupDirectiveParsers(directives);
}
public HtmlMarkupParser HtmlParser { get; set; }
@@ -193,7 +197,7 @@ protected static Func IsSpacingToken(bool includeNewLines, bo
(includeComments && token.Kind == SyntaxKind.CSharpComment);
}
- public override void ParseBlock()
+ public override void ParseBlock1()
{
using (PushSpanConfig(DefaultSpanConfig))
{
@@ -268,8 +272,8 @@ private void AtTransition(SyntaxToken current)
private void AfterTransition()
{
- using (PushSpanConfig(DefaultSpanConfig))
- {
+ //using (PushSpanConfig(DefaultSpanConfig))
+ //{
EnsureCurrent();
try
{
@@ -367,7 +371,7 @@ private void AfterTransition()
// Always put current character back in the buffer for the next parser.
PutCurrentBack();
}
- }
+ //}
}
private void VerbatimBlock()
@@ -553,17 +557,17 @@ private bool MethodCallOrArrayIndex(AcceptedCharactersInternal acceptedCharacter
return false;
}
- protected void CompleteBlock()
+ protected void CompleteBlock1()
{
- CompleteBlock(insertMarkerIfNecessary: true);
+ CompleteBlock1(insertMarkerIfNecessary: true);
}
- protected void CompleteBlock(bool insertMarkerIfNecessary)
+ protected void CompleteBlock1(bool insertMarkerIfNecessary)
{
- CompleteBlock(insertMarkerIfNecessary, captureWhitespaceToEndOfLine: insertMarkerIfNecessary);
+ CompleteBlock1(insertMarkerIfNecessary, captureWhitespaceToEndOfLine: insertMarkerIfNecessary);
}
- protected void CompleteBlock(bool insertMarkerIfNecessary, bool captureWhitespaceToEndOfLine)
+ protected void CompleteBlock1(bool insertMarkerIfNecessary, bool captureWhitespaceToEndOfLine)
{
if (insertMarkerIfNecessary && Context.Builder.LastAcceptedCharacters != AcceptedCharactersInternal.Any)
{
@@ -580,7 +584,7 @@ protected void CompleteBlock(bool insertMarkerIfNecessary, bool captureWhitespac
!Context.DesignTimeMode &&
!IsNested)
{
- CaptureWhitespaceAtEndOfCodeOnlyLine();
+ CaptureWhitespaceAtEndOfCodeOnlyLine1();
}
else
{
@@ -588,7 +592,7 @@ protected void CompleteBlock(bool insertMarkerIfNecessary, bool captureWhitespac
}
}
- private void CaptureWhitespaceAtEndOfCodeOnlyLine()
+ private void CaptureWhitespaceAtEndOfCodeOnlyLine1()
{
var whitespace = ReadWhile(token => token.Kind == SyntaxKind.Whitespace);
if (At(SyntaxKind.NewLine))
@@ -652,7 +656,7 @@ private void ExplicitExpression()
}
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
Span.ChunkGenerator = SpanChunkGenerator.Null;
- CompleteBlock(insertMarkerIfNecessary: false);
+ CompleteBlock1(insertMarkerIfNecessary: false); // This is unnecessary
Output(SpanKindInternal.MetaCode);
}
@@ -675,12 +679,12 @@ private void Template()
private void OtherParserBlock()
{
- ParseWithOtherParser(p => p.ParseBlock());
+ ParseWithOtherParser(p => p.ParseBlock1());
}
private void SectionBlock(string left, string right, bool caseSensitive)
{
- ParseWithOtherParser(p => p.ParseRazorBlock(Tuple.Create(left, right), caseSensitive));
+ ParseWithOtherParser(p => p.ParseRazorBlock1(Tuple.Create(left, right), caseSensitive));
}
private void NestedBlock()
@@ -691,7 +695,7 @@ private void NestedBlock()
IsNested = true;
using (PushSpanConfig())
{
- ParseBlock();
+ ParseBlock1();
}
Span.Start = CurrentLocation;
@@ -700,15 +704,6 @@ private void NestedBlock()
NextToken();
}
- protected override bool IsAtEmbeddedTransition(bool allowTemplatesAndComments, bool allowTransitions)
- {
- // No embedded transitions in C#, so ignore that param
- return allowTemplatesAndComments
- && ((Language.IsTransition(CurrentToken)
- && NextIs(SyntaxKind.LessThan, SyntaxKind.Colon, SyntaxKind.DoubleColon))
- || Language.IsCommentStart(CurrentToken));
- }
-
protected override void HandleEmbeddedTransition()
{
if (Language.IsTransition(CurrentToken))
@@ -770,7 +765,7 @@ protected virtual void ReservedDirective(bool topLevel)
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
Span.ChunkGenerator = SpanChunkGenerator.Null;
Context.Builder.CurrentBlock.Type = BlockKindInternal.Directive;
- CompleteBlock();
+ CompleteBlock1();
Output(SpanKindInternal.MetaCode);
}
@@ -802,7 +797,7 @@ private void DoStatement(bool topLevel)
WhileClause();
if (topLevel)
{
- CompleteBlock();
+ CompleteBlock1();
}
}
@@ -859,7 +854,7 @@ private void UsingKeyword(bool topLevel)
if (topLevel)
{
- CompleteBlock();
+ CompleteBlock1();
}
}
@@ -1070,7 +1065,7 @@ private void TryStatement(bool topLevel)
AfterTryClause();
if (topLevel)
{
- CompleteBlock();
+ CompleteBlock1();
}
}
@@ -1081,7 +1076,7 @@ private void IfStatement(bool topLevel)
AfterIfClause();
if (topLevel)
{
- CompleteBlock();
+ CompleteBlock1();
}
}
@@ -1228,7 +1223,7 @@ private void ConditionalBlock(bool topLevel)
ConditionalBlock(block);
if (topLevel)
{
- CompleteBlock();
+ CompleteBlock1();
}
}
@@ -1353,7 +1348,7 @@ private void HandleStatement(Block block, SyntaxKind type)
switch (type)
{
case SyntaxKind.RazorCommentTransition:
- Output(SpanKindInternal.Code);
+ Output(SpanKindInternal.Code); // Not needed
RazorComment();
Statement(block);
break;
@@ -1645,7 +1640,7 @@ private void HandleDirective(DirectiveDescriptor descriptor)
Output(SpanKindInternal.MetaCode, AcceptedCharactersInternal.None);
// Even if an error was logged do not bail out early. If a directive was used incorrectly it doesn't mean it can't be parsed.
- ValidateDirectiveUsage(descriptor);
+ ValidateDirectiveUsage1(descriptor);
for (var i = 0; i < descriptor.Tokens.Count; i++)
{
@@ -1803,7 +1798,7 @@ private void HandleDirective(DirectiveDescriptor descriptor)
using (PushSpanConfig())
{
- HtmlParser.ParseRazorBlock(Tuple.Create("{", "}"), caseSensitive: true);
+ HtmlParser.ParseRazorBlock1(Tuple.Create("{", "}"), caseSensitive: true);
}
Span.Start = CurrentLocation;
@@ -1847,7 +1842,7 @@ private void HandleDirective(DirectiveDescriptor descriptor)
}
- private void ValidateDirectiveUsage(DirectiveDescriptor descriptor)
+ private void ValidateDirectiveUsage1(DirectiveDescriptor descriptor)
{
if (descriptor.Usage == DirectiveUsage.FileScopedSinglyOccurring)
{
@@ -1902,7 +1897,7 @@ private void ParseDirectiveBlock(DirectiveDescriptor descriptor, Action
{
- var parsedDirective = ParseDirective(lookupText, Span.Start, TagHelperDirectiveType.AddTagHelper, errors);
+ var parsedDirective = ParseDirective1(lookupText, Span.Start, TagHelperDirectiveType.AddTagHelper, errors);
return new AddTagHelperChunkGenerator(
lookupText,
@@ -2072,7 +2067,7 @@ protected virtual void RemoveTagHelperDirective()
SyntaxConstants.CSharp.RemoveTagHelperKeyword,
(lookupText, errors) =>
{
- var parsedDirective = ParseDirective(lookupText, Span.Start, TagHelperDirectiveType.RemoveTagHelper, errors);
+ var parsedDirective = ParseDirective1(lookupText, Span.Start, TagHelperDirectiveType.RemoveTagHelper, errors);
return new RemoveTagHelperChunkGenerator(
lookupText,
@@ -2163,7 +2158,7 @@ private void TagHelperDirective(string keyword, Func, CSharpTransitionSyntax>> _keywordParserMap = new Dictionary, CSharpTransitionSyntax>>();
+ private Dictionary, CSharpTransitionSyntax>> _directiveParserMap = new Dictionary, CSharpTransitionSyntax>>(StringComparer.Ordinal);
+
+ public CSharpCodeBlockSyntax ParseBlock()
+ {
+ if (Context == null)
+ {
+ throw new InvalidOperationException(Resources.Parser_Context_Not_Set);
+ }
+
+ if (EndOfFile)
+ {
+ // Nothing to parse.
+ return null;
+ }
+
+ using (var pooledResult = Pool.Allocate())
+ using (PushSpanContextConfig(DefaultSpanContextConfig))
+ {
+ var builder = pooledResult.Builder;
+ try
+ {
+ NextToken();
+
+ // Unless changed, the block is a statement block
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+ builder.Add(OutputTokensAsStatementLiteral());
+
+ // We are usually called when the other parser sees a transition '@'. Look for it.
+ SyntaxToken transitionToken = null;
+ if (At(SyntaxKind.StringLiteral) &&
+ CurrentToken.Content.Length > 0 &&
+ CurrentToken.Content[0] == SyntaxConstants.TransitionCharacter)
+ {
+ var split = Language.SplitToken(CurrentToken, 1, SyntaxKind.Transition);
+ transitionToken = split.Item1;
+
+ // Back up to the end of the transition
+ Context.Source.Position -= split.Item2.Content.Length;
+ NextToken();
+ }
+ else if (At(SyntaxKind.Transition))
+ {
+ transitionToken = EatCurrentToken();
+ }
+
+ if (transitionToken == null)
+ {
+ transitionToken = SyntaxFactory.MissingToken(SyntaxKind.Transition);
+ }
+
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
+ var transition = GetNodeWithSpanContext(SyntaxFactory.CSharpTransition(transitionToken));
+
+ if (At(SyntaxKind.LeftBrace))
+ {
+ var statementBody = ParseStatementBody();
+ var statement = SyntaxFactory.CSharpStatement(transition, statementBody);
+ builder.Add(statement);
+ }
+ else if (At(SyntaxKind.LeftParenthesis))
+ {
+ var expressionBody = ParseExplicitExpressionBody();
+ var expression = SyntaxFactory.CSharpExpression(transition, expressionBody);
+ builder.Add(expression);
+ }
+ else if (At(SyntaxKind.Identifier))
+ {
+ if (!TryParseDirective(builder, transition, CurrentToken.Content))
+ {
+ if (string.Equals(
+ CurrentToken.Content,
+ SyntaxConstants.CSharp.HelperKeyword,
+ StringComparison.Ordinal))
+ {
+ var diagnostic = RazorDiagnosticFactory.CreateParsing_HelperDirectiveNotAvailable(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length));
+ CurrentToken.SetDiagnostics(new[] { diagnostic });
+ Context.ErrorSink.OnError(diagnostic);
+ }
+
+ var implicitExpressionBody = ParseImplicitExpressionBody();
+ var implicitExpression = SyntaxFactory.CSharpImplicitExpression(transition, implicitExpressionBody);
+ builder.Add(implicitExpression);
+ }
+ }
+ else if (At(SyntaxKind.Keyword))
+ {
+ if (!TryParseDirective(builder, transition, CurrentToken.Content) &&
+ !TryParseKeyword(builder, transition))
+ {
+ // Not a directive or a special keyword. Just parse as an implicit expression.
+ var implicitExpressionBody = ParseImplicitExpressionBody();
+ var implicitExpression = SyntaxFactory.CSharpImplicitExpression(transition, implicitExpressionBody);
+ builder.Add(implicitExpression);
+ }
+
+ builder.Add(OutputTokensAsStatementLiteral());
+ }
+ else
+ {
+ // Invalid character
+ SpanContext.ChunkGenerator = new ExpressionChunkGenerator();
+ SpanContext.EditHandler = new ImplicitExpressionEditHandler(
+ Language.TokenizeString,
+ CurrentKeywords,
+ acceptTrailingDot: IsNested)
+ {
+ AcceptedCharacters = AcceptedCharactersInternal.NonWhitespace
+ };
+
+ AcceptMarkerTokenIfNecessary();
+ var expressionLiteral = SyntaxFactory.CSharpCodeBlock(OutputTokensAsExpressionLiteral());
+ var expressionBody = SyntaxFactory.CSharpImplicitExpressionBody(expressionLiteral);
+ var expressionBlock = SyntaxFactory.CSharpImplicitExpression(transition, expressionBody);
+ builder.Add(expressionBlock);
+
+ if (At(SyntaxKind.Whitespace) || At(SyntaxKind.NewLine))
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_UnexpectedWhiteSpaceAtStartOfCodeBlock(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length)));
+ }
+ else if (EndOfFile)
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_UnexpectedEndOfFileAtStartOfCodeBlock(
+ new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
+ }
+ else
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_UnexpectedCharacterAtStartOfCodeBlock(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length),
+ CurrentToken.Content));
+ }
+ }
+
+ Debug.Assert(TokenBuilder.Count == 0, "We should not have any tokens left.");
+
+ var codeBlock = SyntaxFactory.CSharpCodeBlock(builder.ToList());
+ return codeBlock;
+ }
+ finally
+ {
+ // Always put current character back in the buffer for the next parser.
+ PutCurrentBack();
+ }
+ }
+ }
+
+ private CSharpExpressionBodySyntax ParseExplicitExpressionBody()
+ {
+ var block = new Block(Resources.BlockName_ExplicitExpression, CurrentStart);
+ Assert(SyntaxKind.LeftParenthesis);
+ var leftParenToken = EatCurrentToken();
+ var leftParen = OutputAsMetaCode(leftParenToken);
+
+ using (var pooledResult = Pool.Allocate())
+ {
+ var expressionBuilder = pooledResult.Builder;
+ using (PushSpanContextConfig(ExplicitExpressionSpanContextConfig))
+ {
+ var success = BalanceToken(
+ expressionBuilder,
+ BalancingModes.BacktrackOnFailure |
+ BalancingModes.NoErrorOnFailure |
+ BalancingModes.AllowCommentsAndTemplates,
+ SyntaxKind.LeftParenthesis,
+ SyntaxKind.RightParenthesis,
+ block.Start);
+
+ if (!success)
+ {
+ AcceptTokenUntil(SyntaxKind.LessThan);
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_ExpectedEndOfBlockBeforeEOF(
+ new SourceSpan(block.Start, contentLength: 1 /* ( */), block.Name, ")", "("));
+ }
+
+ // If necessary, put an empty-content marker token here
+ AcceptMarkerTokenIfNecessary();
+ expressionBuilder.Add(OutputTokensAsExpressionLiteral());
+ }
+
+ var expressionBlock = SyntaxFactory.CSharpCodeBlock(expressionBuilder.ToList());
+
+ RazorMetaCodeSyntax rightParen = null;
+ if (At(SyntaxKind.RightParenthesis))
+ {
+ rightParen = OutputAsMetaCode(EatCurrentToken());
+ }
+ else
+ {
+ var missingToken = SyntaxFactory.MissingToken(SyntaxKind.RightParenthesis);
+ rightParen = OutputAsMetaCode(missingToken, SpanContext.EditHandler.AcceptedCharacters);
+ }
+ if (!EndOfFile)
+ {
+ PutCurrentBack();
+ }
+
+ return SyntaxFactory.CSharpExpressionBody(leftParen, expressionBlock, rightParen);
+ }
+ }
+
+ private CSharpImplicitExpressionBodySyntax ParseImplicitExpressionBody(bool async = false)
+ {
+ var accepted = AcceptedCharactersInternal.NonWhitespace;
+ if (async)
+ {
+ // Async implicit expressions include the "await" keyword and therefore need to allow spaces to
+ // separate the "await" and the following code.
+ accepted = AcceptedCharactersInternal.AnyExceptNewline;
+ }
+
+ using (var pooledResult = Pool.Allocate())
+ {
+ var expressionBuilder = pooledResult.Builder;
+ ParseImplicitExpression(expressionBuilder, accepted);
+ var codeBlock = SyntaxFactory.CSharpCodeBlock(expressionBuilder.ToList());
+ return SyntaxFactory.CSharpImplicitExpressionBody(codeBlock);
+ }
+ }
+
+ private void ParseImplicitExpression(in SyntaxListBuilder builder, AcceptedCharactersInternal acceptedCharacters)
+ {
+ using (PushSpanContextConfig(spanContext =>
+ {
+ spanContext.EditHandler = new ImplicitExpressionEditHandler(
+ Language.TokenizeString,
+ Keywords,
+ acceptTrailingDot: IsNested);
+ spanContext.EditHandler.AcceptedCharacters = acceptedCharacters;
+ spanContext.ChunkGenerator = new ExpressionChunkGenerator();
+ }))
+ {
+ do
+ {
+ if (AtIdentifier(allowKeywords: true))
+ {
+ AcceptTokenAndMoveNext();
+ }
+ }
+ while (ParseMethodCallOrArrayIndex(builder, acceptedCharacters));
+
+ PutCurrentBack();
+ builder.Add(OutputTokensAsExpressionLiteral());
+ }
+ }
+
+ private bool ParseMethodCallOrArrayIndex(in SyntaxListBuilder builder, AcceptedCharactersInternal acceptedCharacters)
+ {
+ if (!EndOfFile)
+ {
+ if (CurrentToken.Kind == SyntaxKind.LeftParenthesis ||
+ CurrentToken.Kind == SyntaxKind.LeftBracket)
+ {
+ // If we end within "(", whitespace is fine
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
+
+ SyntaxKind right;
+ bool success;
+
+ using (PushSpanContextConfig((spanContext, prev) =>
+ {
+ prev(spanContext);
+ spanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
+ }))
+ {
+ right = Language.FlipBracket(CurrentToken.Kind);
+ success = BalanceToken(builder, BalancingModes.BacktrackOnFailure | BalancingModes.AllowCommentsAndTemplates);
+ }
+
+ if (!success)
+ {
+ AcceptTokenUntil(SyntaxKind.LessThan);
+ }
+ if (At(right))
+ {
+ AcceptTokenAndMoveNext();
+
+ // At the ending brace, restore the initial accepted characters.
+ SpanContext.EditHandler.AcceptedCharacters = acceptedCharacters;
+ }
+ return ParseMethodCallOrArrayIndex(builder, acceptedCharacters);
+ }
+ if (At(SyntaxKind.QuestionMark))
+ {
+ var next = Lookahead(count: 1);
+
+ if (next != null)
+ {
+ if (next.Kind == SyntaxKind.Dot)
+ {
+ // Accept null conditional dot operator (?.).
+ AcceptTokenAndMoveNext();
+ AcceptTokenAndMoveNext();
+
+ // If the next piece after the ?. is a keyword or identifier then we want to continue.
+ return At(SyntaxKind.Identifier) || At(SyntaxKind.Keyword);
+ }
+ else if (next.Kind == SyntaxKind.LeftBracket)
+ {
+ // We're at the ? for a null conditional bracket operator (?[).
+ AcceptTokenAndMoveNext();
+
+ // Accept the [ and any content inside (it will attempt to balance).
+ return ParseMethodCallOrArrayIndex(builder, acceptedCharacters);
+ }
+ }
+ }
+ else if (At(SyntaxKind.Dot))
+ {
+ var dot = CurrentToken;
+ if (NextToken())
+ {
+ if (At(SyntaxKind.Identifier) || At(SyntaxKind.Keyword))
+ {
+ // Accept the dot and return to the start
+ AcceptToken(dot);
+ return true; // continue
+ }
+ else
+ {
+ // Put the token back
+ PutCurrentBack();
+ }
+ }
+ if (!IsNested)
+ {
+ // Put the "." back
+ PutBack(dot);
+ }
+ else
+ {
+ AcceptToken(dot);
+ }
+ }
+ else if (!At(SyntaxKind.Whitespace) && !At(SyntaxKind.NewLine))
+ {
+ PutCurrentBack();
+ }
+ }
+
+ // Implicit Expression is complete
+ return false;
+ }
+
+ private CSharpStatementBodySyntax ParseStatementBody(Block block = null)
+ {
+ Assert(SyntaxKind.LeftBrace);
+ block = block ?? new Block(Resources.BlockName_Code, CurrentStart);
+ var leftBrace = OutputAsMetaCode(GetExpectedToken(SyntaxKind.LeftBrace));
+ CSharpCodeBlockSyntax codeBlock = null;
+ using (var pooledResult = Pool.Allocate())
+ {
+ var builder = pooledResult.Builder;
+ // Set up auto-complete and parse the code block
+ var editHandler = new AutoCompleteEditHandler(Language.TokenizeString);
+ SpanContext.EditHandler = editHandler;
+ ParseCodeBlock(builder, block, acceptTerminatingBrace: false);
+
+ EnsureCurrent();
+ SpanContext.ChunkGenerator = new StatementChunkGenerator();
+ AcceptMarkerTokenIfNecessary();
+ if (!At(SyntaxKind.RightBrace))
+ {
+ editHandler.AutoCompleteString = "}";
+ }
+ builder.Add(OutputTokensAsStatementLiteral());
+
+ codeBlock = SyntaxFactory.CSharpCodeBlock(builder.ToList());
+ }
+
+ RazorMetaCodeSyntax rightBrace = null;
+ if (At(SyntaxKind.RightBrace))
+ {
+ rightBrace = OutputAsMetaCode(EatCurrentToken());
+ }
+ else
+ {
+ rightBrace = OutputAsMetaCode(
+ SyntaxFactory.MissingToken(SyntaxKind.RightBrace),
+ SpanContext.EditHandler.AcceptedCharacters);
+ }
+
+ if (!IsNested)
+ {
+ EnsureCurrent();
+ if (At(SyntaxKind.NewLine) ||
+ (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.NewLine)))
+ {
+ Context.NullGenerateWhitespaceAndNewLine = true;
+ }
+ }
+
+ return SyntaxFactory.CSharpStatementBody(leftBrace, codeBlock, rightBrace);
+ }
+
+ private void ParseCodeBlock(in SyntaxListBuilder builder, Block block, bool acceptTerminatingBrace = true)
+ {
+ EnsureCurrent();
+ while (!EndOfFile && !At(SyntaxKind.RightBrace))
+ {
+ // Parse a statement, then return here
+ ParseStatement(builder, block: block);
+ EnsureCurrent();
+ }
+
+ if (EndOfFile)
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_ExpectedEndOfBlockBeforeEOF(
+ new SourceSpan(block.Start, contentLength: 1 /* { OR } */), block.Name, "}", "{"));
+ }
+
+ if (acceptTerminatingBrace)
+ {
+ Assert(SyntaxKind.RightBrace);
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
+ AcceptTokenAndMoveNext();
+ }
+ }
+
+ private void ParseStatement(in SyntaxListBuilder builder, Block block)
+ {
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
+ // Accept whitespace but always keep the last whitespace node so we can put it back if necessary
+ var lastWhitespace = AcceptWhitespaceTokensInLines();
+ if (EndOfFile)
+ {
+ if (lastWhitespace != null)
+ {
+ AcceptToken(lastWhitespace);
+ }
+
+ builder.Add(OutputTokensAsStatementLiteral());
+ return;
+ }
+
+ var kind = CurrentToken.Kind;
+ var location = CurrentStart;
+
+ // Both cases @: and @:: are triggered as markup, second colon in second case will be triggered as a plain text
+ var isSingleLineMarkup = kind == SyntaxKind.Transition &&
+ (NextIs(SyntaxKind.Colon, SyntaxKind.DoubleColon));
+
+ var isMarkup = isSingleLineMarkup ||
+ kind == SyntaxKind.LessThan ||
+ (kind == SyntaxKind.Transition && NextIs(SyntaxKind.LessThan));
+
+ if (Context.DesignTimeMode || !isMarkup)
+ {
+ // CODE owns whitespace, MARKUP owns it ONLY in DesignTimeMode.
+ if (lastWhitespace != null)
+ {
+ AcceptToken(lastWhitespace);
+ }
+ }
+ else
+ {
+ var nextToken = Lookahead(1);
+
+ // MARKUP owns whitespace EXCEPT in DesignTimeMode.
+ PutCurrentBack();
+
+ // Put back the whitespace unless it precedes a '' tag.
+ if (nextToken != null &&
+ !string.Equals(nextToken.Content, SyntaxConstants.TextTagName, StringComparison.Ordinal))
+ {
+ PutBack(lastWhitespace);
+ }
+ else
+ {
+ // If it precedes a '' tag, it should be accepted as code.
+ AcceptToken(lastWhitespace);
+ }
+ }
+
+ if (isMarkup)
+ {
+ if (kind == SyntaxKind.Transition && !isSingleLineMarkup)
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_AtInCodeMustBeFollowedByColonParenOrIdentifierStart(
+ new SourceSpan(location, contentLength: 1 /* @ */)));
+ }
+
+ // Markup block
+ builder.Add(OutputTokensAsStatementLiteral());
+ if (Context.DesignTimeMode && CurrentToken != null &&
+ (CurrentToken.Kind == SyntaxKind.LessThan || CurrentToken.Kind == SyntaxKind.Transition))
+ {
+ PutCurrentBack();
+ }
+ OtherParserBlock(builder);
+ }
+ else
+ {
+ // What kind of statement is this?
+ switch (kind)
+ {
+ case SyntaxKind.RazorCommentTransition:
+ AcceptMarkerTokenIfNecessary();
+ builder.Add(OutputTokensAsStatementLiteral());
+ var comment = ParseRazorComment();
+ builder.Add(comment);
+ ParseStatement(builder, block);
+ break;
+ case SyntaxKind.LeftBrace:
+ // Verbatim Block
+ AcceptTokenAndMoveNext();
+ ParseCodeBlock(builder, block);
+ break;
+ case SyntaxKind.Keyword:
+ if (!TryParseKeyword(builder, transition: null))
+ {
+ ParseStandardStatement(builder);
+ }
+ break;
+ case SyntaxKind.Transition:
+ // Embedded Expression block
+ ParseEmbeddedExpression(builder);
+ break;
+ case SyntaxKind.RightBrace:
+ // Possible end of Code Block, just run the continuation
+ break;
+ case SyntaxKind.CSharpComment:
+ AcceptToken(CurrentToken);
+ NextToken();
+ break;
+ default:
+ // Other statement
+ ParseStandardStatement(builder);
+ break;
+ }
+ }
+ }
+
+ private void ParseEmbeddedExpression(in SyntaxListBuilder builder)
+ {
+ // First, verify the type of the block
+ Assert(SyntaxKind.Transition);
+ var transition = CurrentToken;
+ NextToken();
+
+ if (At(SyntaxKind.Transition))
+ {
+ // Escaped "@"
+ builder.Add(OutputTokensAsStatementLiteral());
+
+ // Output "@" as hidden span
+ AcceptToken(transition);
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ builder.Add(OutputTokensAsHiddenLiteral());
+
+ Assert(SyntaxKind.Transition);
+ AcceptTokenAndMoveNext();
+ ParseStandardStatement(builder);
+ }
+ else
+ {
+ // Throw errors as necessary, but continue parsing
+ if (At(SyntaxKind.LeftBrace))
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_UnexpectedNestedCodeBlock(
+ new SourceSpan(CurrentStart, contentLength: 1 /* { */)));
+ }
+
+ // @( or @foo - Nested expression, parse a child block
+ PutCurrentBack();
+ PutBack(transition);
+
+ // Before exiting, add a marker span if necessary
+ AcceptMarkerTokenIfNecessary();
+ builder.Add(OutputTokensAsStatementLiteral());
+
+ var nestedBlock = ParseNestedBlock();
+ builder.Add(nestedBlock);
+ }
+ }
+
+ private RazorSyntaxNode ParseNestedBlock()
+ {
+ var wasNested = IsNested;
+ IsNested = true;
+
+ RazorSyntaxNode nestedBlock;
+ using (PushSpanContextConfig())
+ {
+ nestedBlock = ParseBlock();
+ }
+
+ InitializeContext(SpanContext);
+ IsNested = wasNested;
+ NextToken();
+
+ return nestedBlock;
+ }
+
+ private void ParseStandardStatement(in SyntaxListBuilder builder)
+ {
+ while (!EndOfFile)
+ {
+ var bookmark = CurrentStart.AbsoluteIndex;
+ var read = ReadWhile(token =>
+ token.Kind != SyntaxKind.Semicolon &&
+ token.Kind != SyntaxKind.RazorCommentTransition &&
+ token.Kind != SyntaxKind.Transition &&
+ token.Kind != SyntaxKind.LeftBrace &&
+ token.Kind != SyntaxKind.LeftParenthesis &&
+ token.Kind != SyntaxKind.LeftBracket &&
+ token.Kind != SyntaxKind.RightBrace);
+
+ if (At(SyntaxKind.LeftBrace) ||
+ At(SyntaxKind.LeftParenthesis) ||
+ At(SyntaxKind.LeftBracket))
+ {
+ AcceptToken(read);
+ if (BalanceToken(builder, BalancingModes.AllowCommentsAndTemplates | BalancingModes.BacktrackOnFailure))
+ {
+ OptionalToken(SyntaxKind.RightBrace);
+ }
+ else
+ {
+ // Recovery
+ AcceptTokenUntil(SyntaxKind.LessThan, SyntaxKind.RightBrace);
+ return;
+ }
+ }
+ else if (At(SyntaxKind.Transition) && (NextIs(SyntaxKind.LessThan, SyntaxKind.Colon)))
+ {
+ AcceptToken(read);
+ builder.Add(OutputTokensAsStatementLiteral());
+ ParseTemplate(builder);
+ }
+ else if (At(SyntaxKind.RazorCommentTransition))
+ {
+ AcceptToken(read);
+ AcceptMarkerTokenIfNecessary();
+ builder.Add(OutputTokensAsStatementLiteral());
+ builder.Add(ParseRazorComment());
+ }
+ else if (At(SyntaxKind.Semicolon))
+ {
+ AcceptToken(read);
+ AcceptTokenAndMoveNext();
+ return;
+ }
+ else if (At(SyntaxKind.RightBrace))
+ {
+ AcceptToken(read);
+ return;
+ }
+ else
+ {
+ Context.Source.Position = bookmark;
+ NextToken();
+ AcceptTokenUntil(SyntaxKind.LessThan, SyntaxKind.LeftBrace, SyntaxKind.RightBrace);
+ return;
+ }
+ }
+ }
+
+ private void ParseTemplate(in SyntaxListBuilder builder)
+ {
+ if (Context.InTemplateContext)
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_InlineMarkupBlocksCannotBeNested(
+ new SourceSpan(CurrentStart, contentLength: 1 /* @ */)));
+ }
+ builder.Add(OutputTokensAsStatementLiteral());
+
+ using (var pooledResult = Pool.Allocate())
+ {
+ var templateBuilder = pooledResult.Builder;
+ Context.InTemplateContext = true;
+ PutCurrentBack();
+ OtherParserBlock(templateBuilder);
+
+ var template = SyntaxFactory.CSharpTemplateBlock(templateBuilder.ToList());
+ builder.Add(template);
+
+ Context.InTemplateContext = false;
+ }
+ }
+
+ protected bool TryParseDirective(in SyntaxListBuilder builder, CSharpTransitionSyntax transition, string directive)
+ {
+ if (_directiveParserMap.TryGetValue(directive, out var handler))
+ {
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ handler(builder, transition);
+ return true;
+ }
+
+ return false;
+ }
+
+ private void SetupDirectiveParsers(IEnumerable directiveDescriptors)
+ {
+ var allDirectives = directiveDescriptors.Concat(DefaultDirectiveDescriptors).ToList();
+
+ for (var i = 0; i < allDirectives.Count; i++)
+ {
+ var directiveDescriptor = allDirectives[i];
+ CurrentKeywords.Add(directiveDescriptor.Directive);
+ MapDirectives((builder, transition) => ParseExtensibleDirective(builder, transition, directiveDescriptor), directiveDescriptor.Directive);
+ }
+
+ MapDirectives(ParseTagHelperPrefixDirective, SyntaxConstants.CSharp.TagHelperPrefixKeyword);
+ MapDirectives(ParseAddTagHelperDirective, SyntaxConstants.CSharp.AddTagHelperKeyword);
+ MapDirectives(ParseRemoveTagHelperDirective, SyntaxConstants.CSharp.RemoveTagHelperKeyword);
+ }
+
+ protected void MapDirectives(Action, CSharpTransitionSyntax> handler, params string[] directives)
+ {
+ foreach (var directive in directives)
+ {
+ _directiveParserMap.Add(directive, (builder, transition) =>
+ {
+ handler(builder, transition);
+ Context.SeenDirectives.Add(directive);
+ });
+
+ Keywords.Add(directive);
+
+ // These C# keywords are reserved for use in directives. It's an error to use them outside of
+ // a directive. This code removes the error generation if the directive *is* registered.
+ if (string.Equals(directive, "class", StringComparison.OrdinalIgnoreCase))
+ {
+ _keywordParserMap.Remove(CSharpKeyword.Class);
+ }
+ else if (string.Equals(directive, "namespace", StringComparison.OrdinalIgnoreCase))
+ {
+ _keywordParserMap.Remove(CSharpKeyword.Namespace);
+ }
+ }
+ }
+
+ private void ParseTagHelperPrefixDirective(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ RazorDiagnostic duplicateDiagnostic = null;
+ if (Context.SeenDirectives.Contains(SyntaxConstants.CSharp.TagHelperPrefixKeyword))
+ {
+ var directiveStart = CurrentStart;
+ if (transition != null)
+ {
+ // Start the error from the Transition '@'.
+ directiveStart = new SourceLocation(
+ directiveStart.FilePath,
+ directiveStart.AbsoluteIndex - 1,
+ directiveStart.LineIndex,
+ directiveStart.CharacterIndex - 1);
+ }
+ var errorLength = /* @ */ 1 + SyntaxConstants.CSharp.TagHelperPrefixKeyword.Length;
+ duplicateDiagnostic = RazorDiagnosticFactory.CreateParsing_DuplicateDirective(
+ new SourceSpan(directiveStart, errorLength),
+ SyntaxConstants.CSharp.TagHelperPrefixKeyword);
+ }
+
+ var directiveBody = ParseTagHelperDirective(
+ SyntaxConstants.CSharp.TagHelperPrefixKeyword,
+ (prefix, errors) =>
+ {
+ if (duplicateDiagnostic != null)
+ {
+ errors.Add(duplicateDiagnostic);
+ }
+
+ var parsedDirective = ParseDirective(prefix, CurrentStart, TagHelperDirectiveType.TagHelperPrefix, errors);
+
+ return new TagHelperPrefixDirectiveChunkGenerator(
+ prefix,
+ parsedDirective.DirectiveText,
+ errors);
+ });
+
+ var directive = SyntaxFactory.CSharpDirective(transition, directiveBody);
+ builder.Add(directive);
+ }
+
+ private void ParseAddTagHelperDirective(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ var directiveBody = ParseTagHelperDirective(
+ SyntaxConstants.CSharp.AddTagHelperKeyword,
+ (lookupText, errors) =>
+ {
+ var parsedDirective = ParseDirective(lookupText, CurrentStart, TagHelperDirectiveType.AddTagHelper, errors);
+
+ return new AddTagHelperChunkGenerator(
+ lookupText,
+ parsedDirective.DirectiveText,
+ parsedDirective.TypePattern,
+ parsedDirective.AssemblyName,
+ errors);
+ });
+
+ var directive = SyntaxFactory.CSharpDirective(transition, directiveBody);
+ builder.Add(directive);
+ }
+
+ private void ParseRemoveTagHelperDirective(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ var directiveBody = ParseTagHelperDirective(
+ SyntaxConstants.CSharp.RemoveTagHelperKeyword,
+ (lookupText, errors) =>
+ {
+ var parsedDirective = ParseDirective(lookupText, CurrentStart, TagHelperDirectiveType.RemoveTagHelper, errors);
+
+ return new RemoveTagHelperChunkGenerator(
+ lookupText,
+ parsedDirective.DirectiveText,
+ parsedDirective.TypePattern,
+ parsedDirective.AssemblyName,
+ errors);
+ });
+
+ var directive = SyntaxFactory.CSharpDirective(transition, directiveBody);
+ builder.Add(directive);
+ }
+
+ private CSharpDirectiveBodySyntax ParseTagHelperDirective(
+ string keyword,
+ Func, ISpanChunkGenerator> chunkGeneratorFactory)
+ {
+ AssertDirective(keyword);
+
+ var savedErrorSink = Context.ErrorSink;
+ var directiveErrorSink = new ErrorSink();
+ RazorMetaCodeSyntax keywordBlock = null;
+ using (var pooledResult = Pool.Allocate())
+ {
+ var directiveBuilder = pooledResult.Builder;
+ Context.ErrorSink = directiveErrorSink;
+
+ string directiveValue = null;
+ try
+ {
+ EnsureDirectiveIsAtStartOfLine();
+
+ var keywordStartLocation = CurrentStart;
+
+ // Accept the directive name
+ var keywordToken = EatCurrentToken();
+ var keywordLength = keywordToken.FullWidth + 1 /* @ */;
+
+ var foundWhitespace = At(SyntaxKind.Whitespace);
+
+ // If we found whitespace then any content placed within the whitespace MAY cause a destructive change
+ // to the document. We can't accept it.
+ var acceptedCharacters = foundWhitespace ? AcceptedCharactersInternal.None : AcceptedCharactersInternal.AnyExceptNewline;
+ AcceptToken(keywordToken);
+ keywordBlock = OutputAsMetaCode(OutputTokens(), acceptedCharacters);
+
+ AcceptTokenWhile(SyntaxKind.Whitespace);
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ SpanContext.EditHandler.AcceptedCharacters = acceptedCharacters;
+ directiveBuilder.Add(OutputTokensAsHtmlLiteral());
+
+ if (EndOfFile || At(SyntaxKind.NewLine))
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_DirectiveMustHaveValue(
+ new SourceSpan(keywordStartLocation, keywordLength), keyword));
+
+ directiveValue = string.Empty;
+ }
+ else
+ {
+ // Need to grab the current location before we accept until the end of the line.
+ var startLocation = CurrentStart;
+
+ // Parse to the end of the line. Essentially accepts anything until end of line, comments, invalid code
+ // etc.
+ AcceptTokenUntil(SyntaxKind.NewLine);
+
+ // Pull out the value and remove whitespaces and optional quotes
+ var rawValue = string.Concat(TokenBuilder.ToList().Nodes.Select(s => s.Content)).Trim();
+
+ var startsWithQuote = rawValue.StartsWith("\"", StringComparison.Ordinal);
+ var endsWithQuote = rawValue.EndsWith("\"", StringComparison.Ordinal);
+ if (startsWithQuote != endsWithQuote)
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_IncompleteQuotesAroundDirective(
+ new SourceSpan(startLocation, rawValue.Length), keyword));
+ }
+
+ directiveValue = rawValue;
+ }
+ }
+ finally
+ {
+ SpanContext.ChunkGenerator = chunkGeneratorFactory(directiveValue, directiveErrorSink.Errors.ToList());
+ Context.ErrorSink = savedErrorSink;
+ }
+
+ // Finish the block and output the tokens
+ CompleteBlock();
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.AnyExceptNewline;
+
+ directiveBuilder.Add(OutputTokensAsStatementLiteral());
+ var directiveCodeBlock = SyntaxFactory.CSharpCodeBlock(directiveBuilder.ToList());
+
+ return SyntaxFactory.CSharpDirectiveBody(keywordBlock, directiveCodeBlock);
+ }
+
+ }
+
+ private ParsedDirective ParseDirective(
+ string directiveText,
+ SourceLocation directiveLocation,
+ TagHelperDirectiveType directiveType,
+ List errors)
+ {
+ var offset = 0;
+ directiveText = directiveText.Trim();
+ if (directiveText.Length >= 2 &&
+ directiveText.StartsWith("\"", StringComparison.Ordinal) &&
+ directiveText.EndsWith("\"", StringComparison.Ordinal))
+ {
+ directiveText = directiveText.Substring(1, directiveText.Length - 2);
+ if (string.IsNullOrEmpty(directiveText))
+ {
+ offset = 1;
+ }
+ }
+
+ // If this is the "string literal" form of a directive, we'll need to postprocess the location
+ // and content.
+ //
+ // Ex: @addTagHelper "*, Microsoft.AspNetCore.CoolLibrary"
+ // ^ ^
+ // Start End
+ if (TokenBuilder.Count == 1 &&
+ TokenBuilder[0] is SyntaxToken token &&
+ token.Kind == SyntaxKind.StringLiteral)
+ {
+ offset += token.Content.IndexOf(directiveText, StringComparison.Ordinal);
+
+ // This is safe because inside one of these directives all of the text needs to be on the
+ // same line.
+ var original = directiveLocation;
+ directiveLocation = new SourceLocation(
+ original.FilePath,
+ original.AbsoluteIndex + offset,
+ original.LineIndex,
+ original.CharacterIndex + offset);
+ }
+
+ var parsedDirective = new ParsedDirective()
+ {
+ DirectiveText = directiveText
+ };
+
+ if (directiveType == TagHelperDirectiveType.TagHelperPrefix)
+ {
+ ValidateTagHelperPrefix(parsedDirective.DirectiveText, directiveLocation, errors);
+
+ return parsedDirective;
+ }
+
+ return ParseAddOrRemoveDirective(parsedDirective, directiveLocation, errors);
+ }
+
+ private void ParseExtensibleDirective(in SyntaxListBuilder builder, CSharpTransitionSyntax transition, DirectiveDescriptor descriptor)
+ {
+ AssertDirective(descriptor.Directive);
+
+ var directiveErrorSink = new ErrorSink();
+ var savedErrorSink = Context.ErrorSink;
+ Context.ErrorSink = directiveErrorSink;
+
+ using (var pooledResult = Pool.Allocate())
+ {
+ var directiveBuilder = pooledResult.Builder;
+ var directiveChunkGenerator = new DirectiveChunkGenerator(descriptor);
+ RazorMetaCodeSyntax keywordBlock = null;
+
+ try
+ {
+ EnsureDirectiveIsAtStartOfLine();
+ var directiveStart = CurrentStart;
+ if (transition != null)
+ {
+ // Start the error from the Transition '@'.
+ directiveStart = new SourceLocation(
+ directiveStart.FilePath,
+ directiveStart.AbsoluteIndex - 1,
+ directiveStart.LineIndex,
+ directiveStart.CharacterIndex - 1);
+ }
+
+ AcceptTokenAndMoveNext();
+ keywordBlock = OutputAsMetaCode(OutputTokens());
+
+ // Even if an error was logged do not bail out early. If a directive was used incorrectly it doesn't mean it can't be parsed.
+ ValidateDirectiveUsage(descriptor, directiveStart);
+
+ for (var i = 0; i < descriptor.Tokens.Count; i++)
+ {
+ if (!At(SyntaxKind.Whitespace) &&
+ !At(SyntaxKind.NewLine) &&
+ !EndOfFile)
+ {
+ // This case should never happen in a real scenario. We're just being defensive.
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_DirectiveTokensMustBeSeparatedByWhitespace(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), descriptor.Directive));
+
+ return;
+ }
+
+ var tokenDescriptor = descriptor.Tokens[i];
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
+
+ if (tokenDescriptor.Kind == DirectiveTokenKind.Member ||
+ tokenDescriptor.Kind == DirectiveTokenKind.Namespace ||
+ tokenDescriptor.Kind == DirectiveTokenKind.Type)
+ {
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Whitespace;
+ directiveBuilder.Add(OutputTokensAsStatementLiteral());
+
+ if (EndOfFile || At(SyntaxKind.NewLine))
+ {
+ // Add a marker token to provide CSharp intellisense when we start typing the directive token.
+ AcceptMarkerTokenIfNecessary();
+ SpanContext.ChunkGenerator = new DirectiveTokenChunkGenerator(tokenDescriptor);
+ SpanContext.EditHandler = new DirectiveTokenEditHandler(Language.TokenizeString);
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.NonWhitespace;
+ directiveBuilder.Add(OutputTokensAsStatementLiteral());
+ }
+ }
+ else
+ {
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Whitespace;
+ directiveBuilder.Add(OutputTokensAsHtmlLiteral());
+ }
+
+ if (tokenDescriptor.Optional && (EndOfFile || At(SyntaxKind.NewLine)))
+ {
+ break;
+ }
+ else if (EndOfFile)
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_UnexpectedEOFAfterDirective(
+ new SourceSpan(CurrentStart, contentLength: 1),
+ descriptor.Directive,
+ tokenDescriptor.Kind.ToString().ToLowerInvariant()));
+ return;
+ }
+
+ switch (tokenDescriptor.Kind)
+ {
+ case DirectiveTokenKind.Type:
+ if (!TryParseNamespaceOrTypeName(directiveBuilder))
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_DirectiveExpectsTypeName(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), descriptor.Directive));
+
+ return;
+ }
+ break;
+
+ case DirectiveTokenKind.Namespace:
+ if (!TryParseQualifiedIdentifier(out var identifierLength))
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_DirectiveExpectsNamespace(
+ new SourceSpan(CurrentStart, identifierLength), descriptor.Directive));
+
+ return;
+ }
+ break;
+
+ case DirectiveTokenKind.Member:
+ if (At(SyntaxKind.Identifier))
+ {
+ AcceptTokenAndMoveNext();
+ }
+ else
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_DirectiveExpectsIdentifier(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), descriptor.Directive));
+ return;
+ }
+ break;
+
+ case DirectiveTokenKind.String:
+ if (At(SyntaxKind.StringLiteral) && !CurrentToken.ContainsDiagnostics)
+ {
+ AcceptTokenAndMoveNext();
+ }
+ else
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_DirectiveExpectsQuotedStringLiteral(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), descriptor.Directive));
+ return;
+ }
+ break;
+ }
+
+ SpanContext.ChunkGenerator = new DirectiveTokenChunkGenerator(tokenDescriptor);
+ SpanContext.EditHandler = new DirectiveTokenEditHandler(Language.TokenizeString);
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.NonWhitespace;
+ directiveBuilder.Add(OutputTokensAsStatementLiteral());
+ }
+
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+
+ switch (descriptor.Kind)
+ {
+ case DirectiveKind.SingleLine:
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Whitespace;
+ directiveBuilder.Add(OutputTokensAsNoneLiteral());
+
+ OptionalToken(SyntaxKind.Semicolon);
+ directiveBuilder.Add(OutputAsMetaCode(OutputTokens(), AcceptedCharactersInternal.Whitespace));
+
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
+
+ if (At(SyntaxKind.NewLine))
+ {
+ AcceptTokenAndMoveNext();
+ }
+ else if (!EndOfFile)
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_UnexpectedDirectiveLiteral(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length),
+ descriptor.Directive,
+ Resources.ErrorComponent_Newline));
+ }
+
+
+ // This should contain the optional whitespace after the optional semicolon and the new line.
+ // Output as Markup as we want intellisense here.
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Whitespace;
+ directiveBuilder.Add(OutputTokensAsHtmlLiteral());
+ break;
+ case DirectiveKind.RazorBlock:
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.AllWhitespace;
+ directiveBuilder.Add(OutputTokensAsHtmlLiteral());
+
+ ParseDirectiveBlock(directiveBuilder, descriptor, parseChildren: (childBuilder, startingBraceLocation) =>
+ {
+ // When transitioning to the HTML parser we no longer want to act as if we're in a nested C# state.
+ // For instance, if @hello.
is in a nested C# block we don't want the trailing '.' to be handled
+ // as C#; it should be handled as a period because it's wrapped in markup.
+ var wasNested = IsNested;
+ IsNested = false;
+
+ using (PushSpanContextConfig())
+ {
+ var razorBlock = HtmlParser.ParseRazorBlock(Tuple.Create("{", "}"), caseSensitive: true);
+ directiveBuilder.Add(razorBlock);
+ }
+
+ InitializeContext(SpanContext);
+ IsNested = wasNested;
+ NextToken();
+ });
+ break;
+ case DirectiveKind.CodeBlock:
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.AllWhitespace;
+ directiveBuilder.Add(OutputTokensAsHtmlLiteral());
+
+ ParseDirectiveBlock(directiveBuilder, descriptor, parseChildren: (childBuilder, startingBraceLocation) =>
+ {
+ NextToken();
+ BalanceToken(childBuilder, BalancingModes.NoErrorOnFailure, SyntaxKind.LeftBrace, SyntaxKind.RightBrace, startingBraceLocation);
+ SpanContext.ChunkGenerator = new StatementChunkGenerator();
+ var existingEditHandler = SpanContext.EditHandler;
+ SpanContext.EditHandler = new CodeBlockEditHandler(Language.TokenizeString);
+
+ AcceptMarkerTokenIfNecessary();
+
+ childBuilder.Add(OutputTokensAsStatementLiteral());
+
+ SpanContext.EditHandler = existingEditHandler;
+ });
+ break;
+ }
+ }
+ finally
+ {
+ if (directiveErrorSink.Errors.Count > 0)
+ {
+ directiveChunkGenerator.Diagnostics.AddRange(directiveErrorSink.Errors);
+ }
+
+ Context.ErrorSink = savedErrorSink;
+ }
+
+ directiveBuilder.Add(OutputTokensAsStatementLiteral());
+ var directiveCodeBlock = SyntaxFactory.CSharpCodeBlock(directiveBuilder.ToList());
+
+ var directiveBody = SyntaxFactory.CSharpDirectiveBody(keywordBlock, directiveCodeBlock);
+ var directive = SyntaxFactory.CSharpDirective(transition, directiveBody);
+ directive = (CSharpDirectiveSyntax)directive.SetDiagnostics(directiveErrorSink.Errors.ToArray());
+ builder.Add(directive);
+ }
+ }
+
+ private void ValidateDirectiveUsage(DirectiveDescriptor descriptor, SourceLocation directiveStart)
+ {
+ if (descriptor.Usage == DirectiveUsage.FileScopedSinglyOccurring)
+ {
+ if (Context.SeenDirectives.Contains(descriptor.Directive))
+ {
+ // There will always be at least 1 child because of the `@` transition.
+ var errorLength = /* @ */ 1 + descriptor.Directive.Length;
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_DuplicateDirective(
+ new SourceSpan(directiveStart, errorLength), descriptor.Directive));
+
+ return;
+ }
+ }
+ }
+
+ // Used for parsing a qualified name like that which follows the `namespace` keyword.
+ //
+ // qualified-identifier:
+ // identifier
+ // qualified-identifier . identifier
+ protected bool TryParseQualifiedIdentifier(out int identifierLength)
+ {
+ var currentIdentifierLength = 0;
+ var expectingDot = false;
+ var tokens = ReadWhile(token =>
+ {
+ var type = token.Kind;
+ if ((expectingDot && type == SyntaxKind.Dot) ||
+ (!expectingDot && type == SyntaxKind.Identifier))
+ {
+ expectingDot = !expectingDot;
+ return true;
+ }
+
+ if (type != SyntaxKind.Whitespace &&
+ type != SyntaxKind.NewLine)
+ {
+ expectingDot = false;
+ currentIdentifierLength += token.Content.Length;
+ }
+
+ return false;
+ });
+
+ identifierLength = currentIdentifierLength;
+ var validQualifiedIdentifier = expectingDot;
+ if (validQualifiedIdentifier)
+ {
+ foreach (var token in tokens)
+ {
+ identifierLength += token.Content.Length;
+ AcceptToken(token);
+ }
+
+ return true;
+ }
+ else
+ {
+ PutCurrentBack();
+
+ foreach (var token in tokens)
+ {
+ identifierLength += token.Content.Length;
+ PutBack(token);
+ }
+
+ EnsureCurrent();
+ return false;
+ }
+ }
+
+ private void ParseDirectiveBlock(in SyntaxListBuilder builder, DirectiveDescriptor descriptor, Action, SourceLocation> parseChildren)
+ {
+ if (EndOfFile)
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_UnexpectedEOFAfterDirective(
+ new SourceSpan(CurrentStart, contentLength: 1 /* { */), descriptor.Directive, "{"));
+ }
+ else if (!At(SyntaxKind.LeftBrace))
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_UnexpectedDirectiveLiteral(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), descriptor.Directive, "{"));
+ }
+ else
+ {
+ var editHandler = new AutoCompleteEditHandler(Language.TokenizeString, autoCompleteAtEndOfSpan: true);
+ SpanContext.EditHandler = editHandler;
+ var startingBraceLocation = CurrentStart;
+ AcceptToken(CurrentToken);
+ builder.Add(OutputAsMetaCode(OutputTokens()));
+
+ using (var pooledResult = Pool.Allocate())
+ {
+ var childBuilder = pooledResult.Builder;
+ parseChildren(childBuilder, startingBraceLocation);
+ if (childBuilder.Count > 0)
+ {
+ builder.Add(SyntaxFactory.CSharpCodeBlock(childBuilder.ToList()));
+ }
+ }
+
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ if (!OptionalToken(SyntaxKind.RightBrace))
+ {
+ editHandler.AutoCompleteString = "}";
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_ExpectedEndOfBlockBeforeEOF(
+ new SourceSpan(startingBraceLocation, contentLength: 1 /* } */), descriptor.Directive, "}", "{"));
+
+ AcceptToken(SyntaxFactory.MissingToken(SyntaxKind.RightBrace));
+ }
+ else
+ {
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
+ }
+ CompleteBlock(insertMarkerIfNecessary: false, captureWhitespaceToEndOfLine: true);
+ builder.Add(OutputAsMetaCode(OutputTokens(), SpanContext.EditHandler.AcceptedCharacters));
+ }
+ }
+
+ private bool TryParseKeyword(in SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
+ Debug.Assert(CurrentToken.Kind == SyntaxKind.Keyword && result.HasValue);
+ if (_keywordParserMap.TryGetValue(result.Value, out var handler))
+ {
+ handler(builder, transition);
+ return true;
+ }
+
+ return false;
+ }
+
+ private void SetupExpressionParsers()
+ {
+ MapExpressionKeyword(ParseAwaitExpression, CSharpKeyword.Await);
+ }
+
+ private void SetupKeywordParsers()
+ {
+ MapKeywords(
+ ParseConditionalBlock,
+ CSharpKeyword.For,
+ CSharpKeyword.Foreach,
+ CSharpKeyword.While,
+ CSharpKeyword.Switch,
+ CSharpKeyword.Lock);
+ MapKeywords(ParseCaseStatement, false, CSharpKeyword.Case, CSharpKeyword.Default);
+ MapKeywords(ParseIfStatement, CSharpKeyword.If);
+ MapKeywords(ParseTryStatement, CSharpKeyword.Try);
+ MapKeywords(ParseDoStatement, CSharpKeyword.Do);
+ MapKeywords(ParseUsingKeyword, CSharpKeyword.Using);
+ MapKeywords(ParseReservedDirective, CSharpKeyword.Class, CSharpKeyword.Namespace);
+ }
+
+ private void MapExpressionKeyword(Action, CSharpTransitionSyntax> handler, CSharpKeyword keyword)
+ {
+ _keywordParserMap.Add(keyword, handler);
+
+ // Expression keywords don't belong in the regular keyword list
+ }
+
+ private void MapKeywords(Action, CSharpTransitionSyntax> handler, params CSharpKeyword[] keywords)
+ {
+ MapKeywords(handler, topLevel: true, keywords: keywords);
+ }
+
+ private void MapKeywords(Action, CSharpTransitionSyntax> handler, bool topLevel, params CSharpKeyword[] keywords)
+ {
+ foreach (var keyword in keywords)
+ {
+ _keywordParserMap.Add(keyword, handler);
+ if (topLevel)
+ {
+ Keywords.Add(CSharpLanguageCharacteristics.GetKeyword(keyword));
+ }
+ }
+ }
+
+ private void ParseAwaitExpression(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ // Ensure that we're on the await statement (only runs in debug)
+ Assert(CSharpKeyword.Await);
+
+ // Accept the "await" and move on
+ AcceptTokenAndMoveNext();
+
+ // Accept 1 or more spaces between the await and the following code.
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
+
+ // Top level basically indicates if we're within an expression or statement.
+ // Ex: topLevel true = @await Foo() | topLevel false = @{ await Foo(); }
+ // Note that in this case @{ @await Foo() } top level is true for await.
+ // Therefore, if we're top level then we want to act like an implicit expression,
+ // otherwise just act as whatever we're contained in.
+ var topLevel = transition != null;
+ if (topLevel)
+ {
+ // Setup the Span to be an async implicit expression (an implicit expresison that allows spaces).
+ // Spaces are allowed because of "@await Foo()".
+ var implicitExpressionBody = ParseImplicitExpressionBody(async: true);
+ builder.Add(SyntaxFactory.CSharpImplicitExpression(transition, implicitExpressionBody));
+ }
+ }
+
+ private void ParseConditionalBlock(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ var topLevel = transition != null;
+ ParseConditionalBlock(builder, transition, topLevel);
+ }
+
+ private void ParseConditionalBlock(in SyntaxListBuilder builder, CSharpTransitionSyntax transition, bool topLevel)
+ {
+ Assert(SyntaxKind.Keyword);
+ if (transition != null)
+ {
+ builder.Add(transition);
+ }
+
+ var block = new Block(CurrentToken, CurrentStart);
+ ParseConditionalBlock(builder, block);
+ if (topLevel)
+ {
+ CompleteBlock();
+ }
+ }
+
+ private void ParseConditionalBlock(in SyntaxListBuilder builder, Block block)
+ {
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+
+ // Parse the condition, if present (if not present, we'll let the C# compiler complain)
+ if (TryParseCondition(builder))
+ {
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+
+ ParseExpectedCodeBlock(builder, block);
+ }
+ }
+
+ private bool TryParseCondition(in SyntaxListBuilder builder)
+ {
+ if (At(SyntaxKind.LeftParenthesis))
+ {
+ var complete = BalanceToken(builder, BalancingModes.BacktrackOnFailure | BalancingModes.AllowCommentsAndTemplates);
+ if (!complete)
+ {
+ AcceptTokenUntil(SyntaxKind.NewLine);
+ }
+ else
+ {
+ OptionalToken(SyntaxKind.RightParenthesis);
+ }
+ return complete;
+ }
+ return true;
+ }
+
+ private void ParseExpectedCodeBlock(in SyntaxListBuilder builder, Block block)
+ {
+ if (!EndOfFile)
+ {
+ // Check for "{" to make sure we're at a block
+ if (!At(SyntaxKind.LeftBrace))
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_SingleLineControlFlowStatementsNotAllowed(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length),
+ Language.GetSample(SyntaxKind.LeftBrace),
+ CurrentToken.Content));
+ }
+
+ // Parse the statement and then we're done
+ ParseStatement(builder, block);
+ }
+ }
+
+ private void ParseUnconditionalBlock(in SyntaxListBuilder builder)
+ {
+ Assert(SyntaxKind.Keyword);
+ var block = new Block(CurrentToken, CurrentStart);
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+ ParseExpectedCodeBlock(builder, block);
+ }
+
+ private void ParseCaseStatement(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ Assert(SyntaxKind.Keyword);
+ if (transition != null)
+ {
+ // Normally, case statement won't start with a transition in a valid scenario.
+ // If it does, just accept it and let the compiler complain.
+ builder.Add(transition);
+ }
+ var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
+ Debug.Assert(result.HasValue &&
+ (result.Value == CSharpKeyword.Case ||
+ result.Value == CSharpKeyword.Default));
+ AcceptTokenUntil(SyntaxKind.Colon);
+ OptionalToken(SyntaxKind.Colon);
+ }
+
+ private void ParseIfStatement(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ Assert(CSharpKeyword.If);
+ ParseConditionalBlock(builder, transition, topLevel: false);
+ ParseAfterIfClause(builder);
+ var topLevel = transition != null;
+ if (topLevel)
+ {
+ CompleteBlock();
+ }
+ }
+
+ private void ParseAfterIfClause(SyntaxListBuilder builder)
+ {
+ // Grab whitespace and razor comments
+ var whitespace = SkipToNextImportantToken(builder);
+
+ // Check for an else part
+ if (At(CSharpKeyword.Else))
+ {
+ AcceptToken(whitespace);
+ Assert(CSharpKeyword.Else);
+ ParseElseClause(builder);
+ }
+ else
+ {
+ // No else, return whitespace
+ PutCurrentBack();
+ PutBack(whitespace);
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
+ }
+ }
+
+ private void ParseElseClause(in SyntaxListBuilder builder)
+ {
+ if (!At(CSharpKeyword.Else))
+ {
+ return;
+ }
+ var block = new Block(CurrentToken, CurrentStart);
+
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+ if (At(CSharpKeyword.If))
+ {
+ // ElseIf
+ block.Name = SyntaxConstants.CSharp.ElseIfKeyword;
+ ParseConditionalBlock(builder, block);
+ ParseAfterIfClause(builder);
+ }
+ else if (!EndOfFile)
+ {
+ // Else
+ ParseExpectedCodeBlock(builder, block);
+ }
+ }
+
+ private void ParseTryStatement(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ Assert(CSharpKeyword.Try);
+ var topLevel = transition != null;
+ if (topLevel)
+ {
+ builder.Add(transition);
+ }
+
+ ParseUnconditionalBlock(builder);
+ ParseAfterTryClause(builder);
+ if (topLevel)
+ {
+ CompleteBlock();
+ }
+ }
+
+ private void ParseAfterTryClause(in SyntaxListBuilder builder)
+ {
+ // Grab whitespace
+ var whitespace = SkipToNextImportantToken(builder);
+
+ // Check for a catch or finally part
+ if (At(CSharpKeyword.Catch))
+ {
+ AcceptToken(whitespace);
+ Assert(CSharpKeyword.Catch);
+ ParseFilterableCatchBlock(builder);
+ ParseAfterTryClause(builder);
+ }
+ else if (At(CSharpKeyword.Finally))
+ {
+ AcceptToken(whitespace);
+ Assert(CSharpKeyword.Finally);
+ ParseUnconditionalBlock(builder);
+ }
+ else
+ {
+ // Return whitespace and end the block
+ PutCurrentBack();
+ PutBack(whitespace);
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
+ }
+ }
+
+ private void ParseFilterableCatchBlock(in SyntaxListBuilder builder)
+ {
+ Assert(CSharpKeyword.Catch);
+
+ var block = new Block(CurrentToken, CurrentStart);
+
+ // Accept "catch"
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsValidStatementSpacingToken);
+
+ // Parse the catch condition if present. If not present, let the C# compiler complain.
+ if (TryParseCondition(builder))
+ {
+ AcceptTokenWhile(IsValidStatementSpacingToken);
+
+ if (At(CSharpKeyword.When))
+ {
+ // Accept "when".
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsValidStatementSpacingToken);
+
+ // Parse the filter condition if present. If not present, let the C# compiler complain.
+ if (!TryParseCondition(builder))
+ {
+ // Incomplete condition.
+ return;
+ }
+
+ AcceptTokenWhile(IsValidStatementSpacingToken);
+ }
+
+ ParseExpectedCodeBlock(builder, block);
+ }
+ }
+
+ private void ParseDoStatement(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ Assert(CSharpKeyword.Do);
+ if (transition != null)
+ {
+ builder.Add(transition);
+ }
+
+ ParseUnconditionalBlock(builder);
+ ParseWhileClause(builder);
+ var topLevel = transition != null;
+ if (topLevel)
+ {
+ CompleteBlock();
+ }
+ }
+
+ private void ParseWhileClause(in SyntaxListBuilder builder)
+ {
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
+ var whitespace = SkipToNextImportantToken(builder);
+
+ if (At(CSharpKeyword.While))
+ {
+ AcceptToken(whitespace);
+ Assert(CSharpKeyword.While);
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+ if (TryParseCondition(builder) && OptionalToken(SyntaxKind.Semicolon))
+ {
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
+ }
+ }
+ else
+ {
+ PutCurrentBack();
+ PutBack(whitespace);
+ }
+ }
+
+ private void ParseUsingKeyword(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ Assert(CSharpKeyword.Using);
+ var topLevel = transition != null;
+ var block = new Block(CurrentToken, CurrentStart);
+
+ if (At(SyntaxKind.LeftParenthesis))
+ {
+ // using ( ==> Using Statement
+ ParseUsingStatement(builder, transition, block);
+ }
+ else if (At(SyntaxKind.Identifier) || At(CSharpKeyword.Static))
+ {
+ // using Identifier ==> Using Declaration
+ if (!topLevel)
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_NamespaceImportAndTypeAliasCannotExistWithinCodeBlock(
+ new SourceSpan(block.Start, block.Name.Length)));
+ if (transition != null)
+ {
+ builder.Add(transition);
+ }
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
+ ParseStandardStatement(builder);
+ }
+ else
+ {
+ ParseUsingDeclaration(builder, transition);
+ }
+ }
+ else
+ {
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
+ }
+
+ if (topLevel)
+ {
+ CompleteBlock();
+ }
+ }
+
+ private void ParseUsingStatement(in SyntaxListBuilder builder, CSharpTransitionSyntax transition, Block block)
+ {
+ Assert(CSharpKeyword.Using);
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
+
+ Assert(SyntaxKind.LeftParenthesis);
+ if (transition != null)
+ {
+ builder.Add(transition);
+ }
+
+ // Parse condition
+ if (TryParseCondition(builder))
+ {
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+
+ // Parse code block
+ ParseExpectedCodeBlock(builder, block);
+ }
+ }
+
+ private void ParseUsingDeclaration(in SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ // Using declarations should always be top level. The error case is handled in a different code path.
+ Debug.Assert(transition != null);
+ using (var pooledResult = Pool.Allocate())
+ {
+ var directiveBuilder = pooledResult.Builder;
+ Assert(CSharpKeyword.Using);
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
+ var keywordTokens = OutputTokensAsStatementLiteral();
+ var start = CurrentStart;
+ if (At(SyntaxKind.Identifier))
+ {
+ // non-static using
+ TryParseNamespaceOrTypeName(directiveBuilder);
+ var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+ if (At(SyntaxKind.Assign))
+ {
+ // Alias
+ AcceptToken(whitespace);
+ Assert(SyntaxKind.Assign);
+ AcceptTokenAndMoveNext();
+
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+
+ // One more namespace or type name
+ TryParseNamespaceOrTypeName(directiveBuilder);
+ }
+ else
+ {
+ PutCurrentBack();
+ PutBack(whitespace);
+ }
+ }
+ else if (At(CSharpKeyword.Static))
+ {
+ // static using
+ AcceptTokenAndMoveNext();
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: false, includeComments: true));
+ TryParseNamespaceOrTypeName(directiveBuilder);
+ }
+
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.AnyExceptNewline;
+ SpanContext.ChunkGenerator = new AddImportChunkGenerator(new LocationTagged(
+ string.Concat(TokenBuilder.ToList().Nodes.Select(s => s.Content)),
+ start));
+
+ // Optional ";"
+ if (EnsureCurrent())
+ {
+ OptionalToken(SyntaxKind.Semicolon);
+ }
+
+ directiveBuilder.Add(OutputTokensAsStatementLiteral());
+ var directiveBody = SyntaxFactory.CSharpDirectiveBody(keywordTokens, SyntaxFactory.CSharpCodeBlock(directiveBuilder.ToList()));
+ builder.Add(SyntaxFactory.CSharpDirective(transition, directiveBody));
+ }
+ }
+
+ private bool TryParseNamespaceOrTypeName(in SyntaxListBuilder builder)
+ {
+ if (OptionalToken(SyntaxKind.LeftParenthesis))
+ {
+ while (!OptionalToken(SyntaxKind.RightParenthesis) && !EndOfFile)
+ {
+ OptionalToken(SyntaxKind.Whitespace);
+
+ if (!TryParseNamespaceOrTypeName(builder))
+ {
+ return false;
+ }
+
+ OptionalToken(SyntaxKind.Whitespace);
+ OptionalToken(SyntaxKind.Identifier);
+ OptionalToken(SyntaxKind.Whitespace);
+ OptionalToken(SyntaxKind.Comma);
+ }
+
+ if (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.QuestionMark))
+ {
+ // Only accept the whitespace if we are going to consume the next token.
+ AcceptTokenAndMoveNext();
+ }
+
+ OptionalToken(SyntaxKind.QuestionMark); // Nullable
+
+ return true;
+ }
+ else if (OptionalToken(SyntaxKind.Identifier) || OptionalToken(SyntaxKind.Keyword))
+ {
+ if (OptionalToken(SyntaxKind.DoubleColon))
+ {
+ if (!OptionalToken(SyntaxKind.Identifier))
+ {
+ OptionalToken(SyntaxKind.Keyword);
+ }
+ }
+ if (At(SyntaxKind.LessThan))
+ {
+ ParseTypeArgumentList(builder);
+ }
+ if (OptionalToken(SyntaxKind.Dot))
+ {
+ TryParseNamespaceOrTypeName(builder);
+ }
+
+ if (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.QuestionMark))
+ {
+ // Only accept the whitespace if we are going to consume the next token.
+ AcceptTokenAndMoveNext();
+ }
+
+ OptionalToken(SyntaxKind.QuestionMark); // Nullable
+
+ if (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.LeftBracket))
+ {
+ // Only accept the whitespace if we are going to consume the next token.
+ AcceptTokenAndMoveNext();
+ }
+
+ while (At(SyntaxKind.LeftBracket))
+ {
+ BalanceToken(builder, BalancingModes.None);
+ if (!OptionalToken(SyntaxKind.RightBracket))
+ {
+ AcceptToken(SyntaxFactory.MissingToken(SyntaxKind.RightBracket));
+ }
+ }
+ return true;
+ }
+ else
+ {
+ return false;
+ }
+ }
+
+ private void ParseTypeArgumentList(in SyntaxListBuilder builder)
+ {
+ Assert(SyntaxKind.LessThan);
+ BalanceToken(builder, BalancingModes.None);
+ if (!OptionalToken(SyntaxKind.GreaterThan))
+ {
+ AcceptToken(SyntaxFactory.MissingToken(SyntaxKind.GreaterThan));
+ }
+ }
+
+ private void ParseReservedDirective(SyntaxListBuilder builder, CSharpTransitionSyntax transition)
+ {
+ Context.ErrorSink.OnError(
+ RazorDiagnosticFactory.CreateParsing_ReservedWord(
+ new SourceSpan(CurrentStart, CurrentToken.Content.Length), CurrentToken.Content));
+
+ AcceptTokenAndMoveNext();
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ CompleteBlock();
+ var keyword = OutputAsMetaCode(OutputTokens());
+ var directiveBody = SyntaxFactory.CSharpDirectiveBody(keyword, cSharpCode: null);
+ var directive = SyntaxFactory.CSharpDirective(transition, directiveBody);
+ builder.Add(directive);
+ }
+
+ protected void CompleteBlock()
+ {
+ CompleteBlock(insertMarkerIfNecessary: true);
+ }
+
+ protected void CompleteBlock(bool insertMarkerIfNecessary)
+ {
+ CompleteBlock(insertMarkerIfNecessary, captureWhitespaceToEndOfLine: insertMarkerIfNecessary);
+ }
+
+ protected void CompleteBlock(bool insertMarkerIfNecessary, bool captureWhitespaceToEndOfLine)
+ {
+ if (insertMarkerIfNecessary && Context.LastAcceptedCharacters != AcceptedCharactersInternal.Any)
+ {
+ AcceptMarkerTokenIfNecessary();
+ }
+
+ EnsureCurrent();
+
+ // Read whitespace, but not newlines
+ // If we're not inserting a marker span, we don't need to capture whitespace
+ if (!Context.WhiteSpaceIsSignificantToAncestorBlock &&
+ captureWhitespaceToEndOfLine &&
+ !Context.DesignTimeMode &&
+ !IsNested)
+ {
+ CaptureWhitespaceAtEndOfCodeOnlyLine();
+ }
+ else
+ {
+ PutCurrentBack();
+ }
+ }
+
+ private IEnumerable SkipToNextImportantToken(in SyntaxListBuilder builder)
+ {
+ while (!EndOfFile)
+ {
+ var whitespace = ReadWhile(IsSpacingToken(includeNewLines: true, includeComments: true));
+ if (At(SyntaxKind.RazorCommentTransition))
+ {
+ AcceptToken(whitespace);
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
+ AcceptMarkerTokenIfNecessary();
+ builder.Add(OutputTokensAsStatementLiteral());
+ var comment = ParseRazorComment();
+ builder.Add(comment);
+ }
+ else
+ {
+ return whitespace;
+ }
+ }
+ return Enumerable.Empty();
+ }
+
+ private void CaptureWhitespaceAtEndOfCodeOnlyLine()
+ {
+ var whitespace = ReadWhile(token => token.Kind == SyntaxKind.Whitespace);
+ if (At(SyntaxKind.NewLine))
+ {
+ AcceptToken(whitespace);
+ AcceptTokenAndMoveNext();
+ PutCurrentBack();
+ }
+ else
+ {
+ PutCurrentBack();
+ PutBack(whitespace);
+ }
+ }
+
+ protected override bool IsAtEmbeddedTransition(bool allowTemplatesAndComments, bool allowTransitions)
+ {
+ // No embedded transitions in C#, so ignore that param
+ return allowTemplatesAndComments
+ && ((Language.IsTransition(CurrentToken)
+ && NextIs(SyntaxKind.LessThan, SyntaxKind.Colon, SyntaxKind.DoubleColon))
+ || Language.IsCommentStart(CurrentToken));
+ }
+
+ protected override void ParseEmbeddedTransition(in SyntaxListBuilder builder)
+ {
+ if (Language.IsTransition(CurrentToken))
+ {
+ PutCurrentBack();
+ ParseTemplate(builder);
+ }
+ else if (Language.IsCommentStart(CurrentToken))
+ {
+ // Output tokens before parsing the comment.
+ AcceptMarkerTokenIfNecessary();
+ builder.Add(OutputTokensAsStatementLiteral());
+ var comment = ParseRazorComment();
+ builder.Add(comment);
+ }
+ }
+
+ private void DefaultSpanContextConfig(SpanContextBuilder spanContext)
+ {
+ spanContext.EditHandler = SpanEditHandler.CreateDefault(Language.TokenizeString);
+ spanContext.ChunkGenerator = new StatementChunkGenerator();
+ }
+
+ private void ExplicitExpressionSpanContextConfig(SpanContextBuilder spanContext)
+ {
+ spanContext.EditHandler = SpanEditHandler.CreateDefault(Language.TokenizeString);
+ spanContext.ChunkGenerator = new ExpressionChunkGenerator();
+ }
+
+ private CSharpStatementLiteralSyntax OutputTokensAsStatementLiteral()
+ {
+ var tokens = OutputTokens();
+ if (tokens.Count == 0)
+ {
+ return null;
+ }
+
+ return GetNodeWithSpanContext(SyntaxFactory.CSharpStatementLiteral(tokens));
+ }
+
+ private CSharpExpressionLiteralSyntax OutputTokensAsExpressionLiteral()
+ {
+ var tokens = OutputTokens();
+ if (tokens.Count == 0)
+ {
+ return null;
+ }
+
+ return GetNodeWithSpanContext(SyntaxFactory.CSharpExpressionLiteral(tokens));
+ }
+
+ private CSharpHiddenLiteralSyntax OutputTokensAsHiddenLiteral()
+ {
+ var tokens = OutputTokens();
+ if (tokens.Count == 0)
+ {
+ return null;
+ }
+
+ return GetNodeWithSpanContext(SyntaxFactory.CSharpHiddenLiteral(tokens));
+ }
+
+ private CSharpNoneLiteralSyntax OutputTokensAsNoneLiteral()
+ {
+ var tokens = OutputTokens();
+ if (tokens.Count == 0)
+ {
+ return null;
+ }
+
+
+ return GetNodeWithSpanContext(SyntaxFactory.CSharpNoneLiteral(tokens));
+ }
+
+ private void OtherParserBlock(in SyntaxListBuilder builder)
+ {
+ // When transitioning to the HTML parser we no longer want to act as if we're in a nested C# state.
+ // For instance, if @hello.
is in a nested C# block we don't want the trailing '.' to be handled
+ // as C#; it should be handled as a period because it's wrapped in markup.
+ var wasNested = IsNested;
+ IsNested = false;
+
+ RazorSyntaxNode htmlBlock = null;
+ using (PushSpanContextConfig())
+ {
+ htmlBlock = HtmlParser.ParseBlock();
+ }
+
+ builder.Add(htmlBlock);
+ InitializeContext(SpanContext);
+
+ IsNested = wasNested;
+ NextToken();
+ }
+ }
+}
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs
index c404183ed..a6a2c270e 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlLanguageCharacteristics.cs
@@ -113,7 +113,7 @@ public override SyntaxKind GetKnownTokenType(KnownTokenType type)
return SyntaxKind.NewLine;
case KnownTokenType.Transition:
return SyntaxKind.Transition;
- case KnownTokenType.WhiteSpace:
+ case KnownTokenType.Whitespace:
return SyntaxKind.Whitespace;
default:
return SyntaxKind.Unknown;
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs
index efc0d2759..a1e86703a 100644
--- a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlMarkupParser.cs
@@ -9,7 +9,7 @@
namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
- internal class HtmlMarkupParser : TokenizerBackedParser
+ internal partial class HtmlMarkupParser : TokenizerBackedParser
{
private const string ScriptTagName = "script";
@@ -54,7 +54,7 @@ public HtmlMarkupParser(ParserContext context)
{
}
- public ParserBase CodeParser { get; set; }
+ public CSharpCodeParser CodeParser { get; set; }
public ISet VoidElements
{
@@ -234,7 +234,7 @@ private void OtherParserBlock()
using (PushSpanConfig())
{
- CodeParser.ParseBlock();
+ CodeParser.ParseBlock1();
}
Span.Start = CurrentLocation;
@@ -275,7 +275,7 @@ private void OptionalBangEscape()
}
}
- public override void ParseBlock()
+ public override void ParseBlock1()
{
if (Context == null)
{
@@ -418,6 +418,11 @@ private void TagBlock(Stack> tags)
Accept(_bufferedOpenAngle);
EndTagBlock(tags, complete: false);
}
+ else if (atSpecialTag && At(SyntaxKind.Bang))
+ {
+ Accept(_bufferedOpenAngle);
+ complete = BangTag();
+ }
else
{
complete = AfterTagStart(tagStart, tags, atSpecialTag, tagBlockWrapper);
@@ -460,7 +465,7 @@ private bool AfterTagStart(SourceLocation tagStart,
case SyntaxKind.ForwardSlash:
// End Tag
return EndTag(tagStart, tags, tagBlockWrapper);
- case SyntaxKind.Bang:
+ case SyntaxKind.Bang: // Dead code. This case will never be hit.
// Comment, CDATA, DOCTYPE, or a parser-escaped HTML tag.
if (atSpecialTag)
{
@@ -516,7 +521,7 @@ private bool BangTag()
while (!EndOfFile)
{
SkipToAndParseCode(SyntaxKind.DoubleHyphen);
- var lastDoubleHyphen = AcceptAllButLastDoubleHyphens();
+ var lastDoubleHyphen = AcceptAllButLastDoubleHyphens1();
if (At(SyntaxKind.CloseAngle))
{
@@ -553,7 +558,7 @@ private bool BangTag()
return false;
}
- protected SyntaxToken AcceptAllButLastDoubleHyphens()
+ protected SyntaxToken AcceptAllButLastDoubleHyphens1()
{
var lastDoubleHyphen = CurrentToken;
AcceptWhile(s =>
@@ -732,7 +737,7 @@ private bool EndTag(SourceLocation tagStart,
tagName = CurrentToken.Content;
}
- var matched = RemoveTag(tags, tagName, tagStart);
+ var matched = RemoveTag1(tags, tagName, tagStart);
if (tags.Count == 0 &&
// Note tagName may contain a '!' escape character. This ensures !text> doesn't match here.
@@ -754,7 +759,7 @@ private bool EndTag(SourceLocation tagStart,
}
}
- private void RecoverTextTag()
+ private void RecoverTextTag1()
{
// We don't want to skip-to and parse because there shouldn't be anything in the body of text tags.
AcceptUntil(SyntaxKind.CloseAngle, SyntaxKind.NewLine);
@@ -781,7 +786,7 @@ private bool EndTextTag(SyntaxToken solidus, IDisposable tagBlockWrapper)
new SourceSpan(textLocation, contentLength: 4 /* text */)));
Span.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
- RecoverTextTag();
+ RecoverTextTag1();
}
else
{
@@ -833,14 +838,14 @@ private void TagContent()
else
{
// We are here ($):
- while (!EndOfFile && !IsEndOfTag())
+ while (!EndOfFile && !IsEndOfTag1())
{
BeforeAttribute();
}
}
}
- private bool IsEndOfTag()
+ private bool IsEndOfTag1()
{
if (At(SyntaxKind.ForwardSlash))
{
@@ -1236,7 +1241,7 @@ private bool StartTag(Stack> tags, IDisposabl
RazorDiagnosticFactory.CreateParsing_TextTagCannotContainAttributes(
new SourceSpan(textLocation, contentLength: 4 /* text */)));
- RecoverTextTag();
+ RecoverTextTag1();
}
else
{
@@ -1353,7 +1358,7 @@ private bool RestOfTag(Tuple tag,
}
else if (string.Equals(tagName, ScriptTagName, StringComparison.OrdinalIgnoreCase))
{
- if (!CurrentScriptTagExpectsHtml())
+ if (!CurrentScriptTagExpectsHtml1())
{
CompleteTagBlockWithSpan(tagBlockWrapper, AcceptedCharactersInternal.None, SpanKindInternal.Markup);
@@ -1465,7 +1470,7 @@ private bool AcceptUntilAll(params SyntaxKind[] endSequence)
return false;
}
- private bool RemoveTag(Stack> tags, string tagName, SourceLocation tagStart)
+ private bool RemoveTag1(Stack> tags, string tagName, SourceLocation tagStart)
{
Tuple currentTag = null;
while (tags.Count > 0)
@@ -1583,7 +1588,7 @@ internal static bool IsValidAttributeNameToken(SyntaxToken token)
tokenType != SyntaxKind.Unknown;
}
- public void ParseDocument()
+ public void ParseDocument1()
{
if (Context == null)
{
@@ -1674,7 +1679,7 @@ private void ScanTagInDocumentContext()
// If the script tag expects javascript content then we should do minimal parsing until we reach
// the end script tag. Don't want to incorrectly parse a "var tag = '';" as an HTML tag.
- if (scriptTag && !CurrentScriptTagExpectsHtml())
+ if (scriptTag && !CurrentScriptTagExpectsHtml1())
{
Output(SpanKindInternal.Markup);
tagBlock.Dispose();
@@ -1705,7 +1710,7 @@ private void ScanTagInDocumentContext()
}
}
- private bool CurrentScriptTagExpectsHtml()
+ private bool CurrentScriptTagExpectsHtml1()
{
var blockBuilder = Context.Builder.CurrentBlock;
@@ -1753,7 +1758,7 @@ private static bool IsTypeAttribute(Block block)
return false;
}
- public void ParseRazorBlock(Tuple nestingSequences, bool caseSensitive)
+ public void ParseRazorBlock1(Tuple nestingSequences, bool caseSensitive)
{
if (Context == null)
{
@@ -1786,9 +1791,9 @@ private void NonNestingSection(string[] nestingSequenceComponents)
{
do
{
- SkipToAndParseCode(token => token.Kind == SyntaxKind.OpenAngle || AtEnd(nestingSequenceComponents));
+ SkipToAndParseCode(token => token.Kind == SyntaxKind.OpenAngle || AtEnd1(nestingSequenceComponents));
ScanTagInDocumentContext();
- if (!EndOfFile && AtEnd(nestingSequenceComponents))
+ if (!EndOfFile && AtEnd1(nestingSequenceComponents))
{
break;
}
@@ -1825,7 +1830,7 @@ private void NestingSection(Tuple nestingSequences)
}
}
- private bool AtEnd(string[] nestingSequenceComponents)
+ private bool AtEnd1(string[] nestingSequenceComponents)
{
EnsureCurrent();
if (string.Equals(CurrentToken.Content, nestingSequenceComponents[0], Comparison))
diff --git a/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlParser.cs b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlParser.cs
new file mode 100644
index 000000000..707f56334
--- /dev/null
+++ b/src/Microsoft.AspNetCore.Razor.Language/Legacy/HtmlParser.cs
@@ -0,0 +1,1741 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
+
+namespace Microsoft.AspNetCore.Razor.Language.Legacy
+{
+ internal partial class HtmlMarkupParser
+ {
+ public HtmlDocumentSyntax ParseDocument()
+ {
+ if (Context == null)
+ {
+ throw new InvalidOperationException(Resources.Parser_Context_Not_Set);
+ }
+
+ using (var pooledResult = Pool.Allocate())
+ using (PushSpanContextConfig(DefaultMarkupSpanContext))
+ {
+ var builder = pooledResult.Builder;
+ NextToken();
+ while (!EndOfFile)
+ {
+ SkipToAndParseCode(builder, SyntaxKind.OpenAngle);
+ ParseTagInDocumentContext(builder);
+ }
+ AcceptMarkerTokenIfNecessary();
+ builder.Add(OutputTokensAsHtmlLiteral());
+
+ var markup = SyntaxFactory.HtmlMarkupBlock(builder.ToList());
+
+ return SyntaxFactory.HtmlDocument(markup);
+ }
+ }
+
+ private void SkipToAndParseCode(in SyntaxListBuilder builder, SyntaxKind type)
+ {
+ SkipToAndParseCode(builder, token => token.Kind == type);
+ }
+
+ private void SkipToAndParseCode(in SyntaxListBuilder builder, Func condition)
+ {
+ SyntaxToken last = null;
+ var startOfLine = false;
+ while (!EndOfFile && !condition(CurrentToken))
+ {
+ if (Context.NullGenerateWhitespaceAndNewLine)
+ {
+ Context.NullGenerateWhitespaceAndNewLine = false;
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ AcceptTokenWhile(token => token.Kind == SyntaxKind.Whitespace);
+ if (At(SyntaxKind.NewLine))
+ {
+ AcceptTokenAndMoveNext();
+ }
+
+ builder.Add(OutputTokensAsHtmlLiteral());
+ }
+ else if (At(SyntaxKind.NewLine))
+ {
+ if (last != null)
+ {
+ AcceptToken(last);
+ }
+
+ // Mark the start of a new line
+ startOfLine = true;
+ last = null;
+ AcceptTokenAndMoveNext();
+ }
+ else if (At(SyntaxKind.Transition))
+ {
+ var transition = CurrentToken;
+ NextToken();
+ if (At(SyntaxKind.Transition))
+ {
+ if (last != null)
+ {
+ AcceptToken(last);
+ last = null;
+ }
+ builder.Add(OutputTokensAsHtmlLiteral());
+ AcceptToken(transition);
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ builder.Add(OutputTokensAsHtmlLiteral());
+ AcceptTokenAndMoveNext();
+ continue; // while
+ }
+ else
+ {
+ if (!EndOfFile)
+ {
+ PutCurrentBack();
+ }
+ PutBack(transition);
+ }
+
+ // Handle whitespace rewriting
+ if (last != null)
+ {
+ if (!Context.DesignTimeMode && last.Kind == SyntaxKind.Whitespace && startOfLine)
+ {
+ // Put the whitespace back too
+ startOfLine = false;
+ PutBack(last);
+ last = null;
+ }
+ else
+ {
+ // Accept last
+ AcceptToken(last);
+ last = null;
+ }
+ }
+
+ OtherParserBlock(builder);
+ }
+ else if (At(SyntaxKind.RazorCommentTransition))
+ {
+ if (last != null)
+ {
+ // Don't render the whitespace between the start of the line and the razor comment.
+ if (startOfLine && last.Kind == SyntaxKind.Whitespace)
+ {
+ AcceptMarkerTokenIfNecessary();
+ // Output the tokens that may have been accepted prior to the whitespace.
+ builder.Add(OutputTokensAsHtmlLiteral());
+
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ }
+
+ AcceptToken(last);
+ last = null;
+ }
+
+ AcceptMarkerTokenIfNecessary();
+ builder.Add(OutputTokensAsHtmlLiteral());
+
+ var comment = ParseRazorComment();
+ builder.Add(comment);
+
+ // Handle the whitespace and newline at the end of a razor comment.
+ if (startOfLine &&
+ (At(SyntaxKind.NewLine) ||
+ (At(SyntaxKind.Whitespace) && NextIs(SyntaxKind.NewLine))))
+ {
+ AcceptTokenWhile(IsSpacingToken(includeNewLines: false));
+ AcceptTokenAndMoveNext();
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ builder.Add(OutputTokensAsHtmlLiteral());
+ }
+ }
+ else
+ {
+ // As long as we see whitespace, we're still at the "start" of the line
+ startOfLine &= At(SyntaxKind.Whitespace);
+
+ // If there's a last token, accept it
+ if (last != null)
+ {
+ AcceptToken(last);
+ last = null;
+ }
+
+ // Advance
+ last = CurrentToken;
+ NextToken();
+ }
+ }
+
+ if (last != null)
+ {
+ AcceptToken(last);
+ }
+ }
+
+ ///
+ /// Reads the content of a tag (if present) in the MarkupDocument (or MarkupSection) context,
+ /// where we don't care about maintaining a stack of tags.
+ ///
+ private void ParseTagInDocumentContext(in SyntaxListBuilder builder)
+ {
+ if (At(SyntaxKind.OpenAngle))
+ {
+ if (NextIs(SyntaxKind.Bang))
+ {
+ // Checking to see if we meet the conditions of a special '!' tag: ())
+ {
+ var tagBuilder = pooledResult.Builder;
+ AcceptTokenAndMoveNext(); // Accept '<'
+
+ if (!At(SyntaxKind.ForwardSlash))
+ {
+ ParseOptionalBangEscape(tagBuilder);
+
+ // Parsing a start tag
+ var scriptTag = At(SyntaxKind.Text) &&
+ string.Equals(CurrentToken.Content, "script", StringComparison.OrdinalIgnoreCase);
+ OptionalToken(SyntaxKind.Text);
+ ParseTagContent(tagBuilder); // Parse the tag, don't care about the content
+ OptionalToken(SyntaxKind.ForwardSlash);
+ OptionalToken(SyntaxKind.CloseAngle);
+
+ // If the script tag expects javascript content then we should do minimal parsing until we reach
+ // the end script tag. Don't want to incorrectly parse a "var tag = '';" as an HTML tag.
+ if (scriptTag && !CurrentScriptTagExpectsHtml(builder))
+ {
+ tagBuilder.Add(OutputTokensAsHtmlLiteral());
+ var block = SyntaxFactory.HtmlTagBlock(tagBuilder.ToList());
+ builder.Add(block);
+
+ SkipToEndScriptAndParseCode(builder);
+ return;
+ }
+ }
+ else
+ {
+ // Parsing an end tag
+ // This section can accept things like: '
' or '' etc.
+ ParserState = ParserState.EndTag;
+ OptionalToken(SyntaxKind.ForwardSlash);
+
+ // Whitespace here is invalid (according to the spec)
+ ParseOptionalBangEscape(tagBuilder);
+ OptionalToken(SyntaxKind.Text);
+ OptionalToken(SyntaxKind.Whitespace);
+ OptionalToken(SyntaxKind.CloseAngle);
+ ParserState = ParserState.Content;
+ }
+
+ tagBuilder.Add(OutputTokensAsHtmlLiteral());
+
+ // End tag block
+ var tagBlock = SyntaxFactory.HtmlTagBlock(tagBuilder.ToList());
+ builder.Add(tagBlock);
+ }
+ }
+ }
+
+ private void ParseTagContent(in SyntaxListBuilder builder)
+ {
+ if (!At(SyntaxKind.Whitespace) && !At(SyntaxKind.NewLine))
+ {
+ // We should be right after the tag name, so if there's no whitespace or new line, something is wrong
+ RecoverToEndOfTag(builder);
+ }
+ else
+ {
+ // We are here ($):
+ while (!EndOfFile && !IsEndOfTag())
+ {
+ BeforeAttribute(builder);
+ }
+ }
+ }
+
+ private bool IsEndOfTag()
+ {
+ if (At(SyntaxKind.ForwardSlash))
+ {
+ if (NextIs(SyntaxKind.CloseAngle))
+ {
+ return true;
+ }
+ else
+ {
+ AcceptTokenAndMoveNext();
+ }
+ }
+ return At(SyntaxKind.CloseAngle) || At(SyntaxKind.OpenAngle);
+ }
+
+ private void BeforeAttribute(in SyntaxListBuilder builder)
+ {
+ // http://dev.w3.org/html5/spec/tokenization.html#before-attribute-name-state
+ // Capture whitespace
+ var whitespace = ReadWhile(token => token.Kind == SyntaxKind.Whitespace || token.Kind == SyntaxKind.NewLine);
+
+ if (At(SyntaxKind.Transition) || At(SyntaxKind.RazorCommentTransition))
+ {
+ // Transition outside of attribute value => Switch to recovery mode
+ AcceptToken(whitespace);
+ RecoverToEndOfTag(builder);
+ return;
+ }
+
+ // http://dev.w3.org/html5/spec/tokenization.html#attribute-name-state
+ // Read the 'name' (i.e. read until the '=' or whitespace/newline)
+ var nameTokens = Enumerable.Empty();
+ var whitespaceAfterAttributeName = Enumerable.Empty();
+ if (IsValidAttributeNameToken(CurrentToken))
+ {
+ nameTokens = ReadWhile(token =>
+ token.Kind != SyntaxKind.Whitespace &&
+ token.Kind != SyntaxKind.NewLine &&
+ token.Kind != SyntaxKind.Equals &&
+ token.Kind != SyntaxKind.CloseAngle &&
+ token.Kind != SyntaxKind.OpenAngle &&
+ (token.Kind != SyntaxKind.ForwardSlash || !NextIs(SyntaxKind.CloseAngle)));
+
+ // capture whitespace after attribute name (if any)
+ whitespaceAfterAttributeName = ReadWhile(
+ token => token.Kind == SyntaxKind.Whitespace || token.Kind == SyntaxKind.NewLine);
+ }
+ else
+ {
+ // Unexpected character in tag, enter recovery
+ AcceptToken(whitespace);
+ RecoverToEndOfTag(builder);
+ return;
+ }
+
+ if (!At(SyntaxKind.Equals))
+ {
+ // Minimized attribute
+
+ // We are at the prefix of the next attribute or the end of tag. Put it back so it is parsed later.
+ PutCurrentBack();
+ PutBack(whitespaceAfterAttributeName);
+
+ // Output anything prior to the attribute, in most cases this will be the tag name:
+ // |. If in-between other attributes this will noop or output malformed attribute
+ // content (if the previous attribute was malformed).
+ builder.Add(OutputTokensAsHtmlLiteral());
+
+ AcceptToken(whitespace);
+ var namePrefix = OutputTokensAsHtmlLiteral();
+ AcceptToken(nameTokens);
+ var name = OutputTokensAsHtmlLiteral();
+
+ var minimizedAttributeBlock = SyntaxFactory.HtmlMinimizedAttributeBlock(namePrefix, name);
+ builder.Add(minimizedAttributeBlock);
+
+ return;
+ }
+
+ // Not a minimized attribute, parse as if it were well-formed (if attribute turns out to be malformed we
+ // will go into recovery).
+ builder.Add(OutputTokensAsHtmlLiteral());
+
+ var attributeBlock = ParseAttributePrefix(whitespace, nameTokens, whitespaceAfterAttributeName);
+
+ builder.Add(attributeBlock);
+ }
+
+ private HtmlAttributeBlockSyntax ParseAttributePrefix(
+ IEnumerable whitespace,
+ IEnumerable nameTokens,
+ IEnumerable whitespaceAfterAttributeName)
+ {
+ // First, determine if this is a 'data-' attribute (since those can't use conditional attributes)
+ var nameContent = string.Concat(nameTokens.Select(s => s.Content));
+ var attributeCanBeConditional =
+ Context.FeatureFlags.EXPERIMENTAL_AllowConditionalDataDashAttributes ||
+ !nameContent.StartsWith("data-", StringComparison.OrdinalIgnoreCase);
+
+ // Accept the whitespace and name
+ AcceptToken(whitespace);
+ var namePrefix = OutputTokensAsHtmlLiteral();
+ AcceptToken(nameTokens);
+ var name = OutputTokensAsHtmlLiteral();
+
+ // Since this is not a minimized attribute, the whitespace after attribute name belongs to this attribute.
+ AcceptToken(whitespaceAfterAttributeName);
+ var nameSuffix = OutputTokensAsHtmlLiteral();
+ Assert(SyntaxKind.Equals); // We should be at "="
+ var equalsToken = EatCurrentToken();
+
+ var whitespaceAfterEquals = ReadWhile(token => token.Kind == SyntaxKind.Whitespace || token.Kind == SyntaxKind.NewLine);
+ var quote = SyntaxKind.Unknown;
+ if (At(SyntaxKind.SingleQuote) || At(SyntaxKind.DoubleQuote))
+ {
+ // Found a quote, the whitespace belongs to this attribute.
+ AcceptToken(whitespaceAfterEquals);
+ quote = CurrentToken.Kind;
+ AcceptTokenAndMoveNext();
+ }
+ else if (whitespaceAfterEquals.Any())
+ {
+ // No quotes found after the whitespace. Put it back so that it can be parsed later.
+ PutCurrentBack();
+ PutBack(whitespaceAfterEquals);
+ }
+
+ HtmlTextLiteralSyntax valuePrefix = null;
+ HtmlBlockSyntax attributeValue = null;
+ HtmlTextLiteralSyntax valueSuffix = null;
+
+ if (attributeCanBeConditional)
+ {
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null; // The block chunk generator will render the prefix
+
+ // We now have the value prefix which is usually whitespace and/or a quote
+ valuePrefix = OutputTokensAsHtmlLiteral();
+
+ // Read the attribute value only if the value is quoted
+ // or if there is no whitespace between '=' and the unquoted value.
+ if (quote != SyntaxKind.Unknown || !whitespaceAfterEquals.Any())
+ {
+ using (var pooledResult = Pool.Allocate())
+ {
+ var attributeValueBuilder = pooledResult.Builder;
+ // Read the attribute value.
+ while (!EndOfFile && !IsEndOfAttributeValue(quote, CurrentToken))
+ {
+ ParseAttributeValue(attributeValueBuilder, quote);
+ }
+
+ attributeValue = SyntaxFactory.HtmlBlock(attributeValueBuilder.ToList());
+ }
+ }
+
+ // Capture the suffix
+ if (quote != SyntaxKind.Unknown && At(quote))
+ {
+ AcceptTokenAndMoveNext();
+ // Again, block chunk generator will render the suffix
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ valueSuffix = OutputTokensAsHtmlLiteral();
+ }
+ }
+ else if (quote != SyntaxKind.Unknown || !whitespaceAfterEquals.Any())
+ {
+ valuePrefix = OutputTokensAsHtmlLiteral();
+
+ using (var pooledResult = Pool.Allocate())
+ {
+ var attributeValueBuilder = pooledResult.Builder;
+ // Not a "conditional" attribute, so just read the value
+ SkipToAndParseCode(attributeValueBuilder, token => IsEndOfAttributeValue(quote, token));
+
+ // Capture the attribute value (will include everything in-between the attribute's quotes).
+ attributeValue = SyntaxFactory.HtmlBlock(attributeValueBuilder.ToList());
+ }
+
+ if (quote != SyntaxKind.Unknown)
+ {
+ OptionalToken(quote);
+ valueSuffix = OutputTokensAsHtmlLiteral();
+ }
+ }
+ else
+ {
+ // There is no quote and there is whitespace after equals. There is no attribute value.
+ }
+
+ return SyntaxFactory.HtmlAttributeBlock(namePrefix, name, nameSuffix, equalsToken, valuePrefix, attributeValue, valueSuffix);
+ }
+
+ private void ParseAttributeValue(in SyntaxListBuilder builder, SyntaxKind quote)
+ {
+ var prefixStart = CurrentStart;
+ var prefixTokens = ReadWhile(token => token.Kind == SyntaxKind.Whitespace || token.Kind == SyntaxKind.NewLine);
+
+ if (At(SyntaxKind.Transition))
+ {
+ if (NextIs(SyntaxKind.Transition))
+ {
+ // Wrapping this in a block so that the ConditionalAttributeCollapser doesn't rewrite it.
+ using (var pooledResult = Pool.Allocate())
+ {
+ var markupBuilder = pooledResult.Builder;
+ AcceptToken(prefixTokens);
+
+ // Render a single "@" in place of "@@".
+ SpanContext.ChunkGenerator = new LiteralAttributeChunkGenerator(
+ new LocationTagged(string.Concat(prefixTokens.Select(s => s.Content)), prefixStart),
+ new LocationTagged(CurrentToken.Content, CurrentStart));
+ AcceptTokenAndMoveNext();
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
+ markupBuilder.Add(OutputTokensAsHtmlLiteral());
+
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ AcceptTokenAndMoveNext();
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
+ markupBuilder.Add(OutputTokensAsHtmlLiteral());
+
+ var markupBlock = SyntaxFactory.HtmlMarkupBlock(markupBuilder.ToList());
+ builder.Add(markupBlock);
+ }
+ }
+ else
+ {
+ AcceptToken(prefixTokens);
+ var valueStart = CurrentStart;
+ PutCurrentBack();
+
+ var prefix = OutputTokensAsHtmlLiteral();
+
+ // Dynamic value, start a new block and set the chunk generator
+ using (var pooledResult = Pool.Allocate())
+ {
+ var dynamicAttributeValueBuilder = pooledResult.Builder;
+
+ OtherParserBlock(dynamicAttributeValueBuilder);
+ var value = SyntaxFactory.HtmlDynamicAttributeValue(prefix, SyntaxFactory.HtmlBlock(dynamicAttributeValueBuilder.ToList()));
+ builder.Add(value);
+ }
+ }
+ }
+ else
+ {
+ AcceptToken(prefixTokens);
+ var prefix = OutputTokensAsHtmlLiteral();
+
+ // Literal value
+ // 'quote' should be "Unknown" if not quoted and tokens coming from the tokenizer should never have
+ // "Unknown" type.
+ var valueTokens = ReadWhile(token =>
+ // These three conditions find separators which break the attribute value into portions
+ token.Kind != SyntaxKind.Whitespace &&
+ token.Kind != SyntaxKind.NewLine &&
+ token.Kind != SyntaxKind.Transition &&
+ // This condition checks for the end of the attribute value (it repeats some of the checks above
+ // but for now that's ok)
+ !IsEndOfAttributeValue(quote, token));
+ AcceptToken(valueTokens);
+ var value = OutputTokensAsHtmlLiteral();
+
+ var literalAttributeValue = SyntaxFactory.HtmlLiteralAttributeValue(prefix, value);
+ builder.Add(literalAttributeValue);
+ }
+ }
+
+ private void RecoverToEndOfTag(in SyntaxListBuilder builder)
+ {
+ // Accept until ">", "/" or "<", but parse code
+ while (!EndOfFile)
+ {
+ SkipToAndParseCode(builder, IsTagRecoveryStopPoint);
+ if (!EndOfFile)
+ {
+ EnsureCurrent();
+ switch (CurrentToken.Kind)
+ {
+ case SyntaxKind.SingleQuote:
+ case SyntaxKind.DoubleQuote:
+ ParseQuoted(builder);
+ break;
+ case SyntaxKind.OpenAngle:
+ // Another "<" means this tag is invalid.
+ case SyntaxKind.ForwardSlash:
+ // Empty tag
+ case SyntaxKind.CloseAngle:
+ // End of tag
+ return;
+ default:
+ AcceptTokenAndMoveNext();
+ break;
+ }
+ }
+ }
+ }
+
+ private void ParseQuoted(in SyntaxListBuilder builder)
+ {
+ var type = CurrentToken.Kind;
+ AcceptTokenAndMoveNext();
+ ParseQuoted(builder, type);
+ }
+
+ private void ParseQuoted(in SyntaxListBuilder builder, SyntaxKind type)
+ {
+ SkipToAndParseCode(builder, type);
+ if (!EndOfFile)
+ {
+ Assert(type);
+ AcceptTokenAndMoveNext();
+ }
+ }
+
+ private bool ParseBangTag(in SyntaxListBuilder builder)
+ {
+ // Accept "!"
+ Assert(SyntaxKind.Bang);
+
+ if (AcceptTokenAndMoveNext())
+ {
+ if (IsHtmlCommentAhead())
+ {
+ using (var pooledResult = Pool.Allocate())
+ {
+ var htmlCommentBuilder = pooledResult.Builder;
+
+ // Accept the double-hyphen token at the beginning of the comment block.
+ AcceptTokenAndMoveNext();
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
+ htmlCommentBuilder.Add(OutputTokensAsHtmlLiteral());
+
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Whitespace;
+ while (!EndOfFile)
+ {
+ SkipToAndParseCode(htmlCommentBuilder, SyntaxKind.DoubleHyphen);
+ var lastDoubleHyphen = AcceptAllButLastDoubleHyphens();
+
+ if (At(SyntaxKind.CloseAngle))
+ {
+ // Output the content in the comment block as a separate markup
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Whitespace;
+ htmlCommentBuilder.Add(OutputTokensAsHtmlLiteral());
+
+ // This is the end of a comment block
+ AcceptToken(lastDoubleHyphen);
+ AcceptTokenAndMoveNext();
+ SpanContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
+ htmlCommentBuilder.Add(OutputTokensAsHtmlLiteral());
+ var commentBlock = SyntaxFactory.HtmlCommentBlock(htmlCommentBuilder.ToList());
+ builder.Add(commentBlock);
+ return true;
+ }
+ else if (lastDoubleHyphen != null)
+ {
+ AcceptToken(lastDoubleHyphen);
+ }
+ }
+ }
+ }
+ else if (CurrentToken.Kind == SyntaxKind.LeftBracket)
+ {
+ if (AcceptTokenAndMoveNext())
+ {
+ return TryParseCData(builder);
+ }
+ }
+ else
+ {
+ AcceptTokenAndMoveNext();
+ return AcceptTokenUntilAll(builder, SyntaxKind.CloseAngle);
+ }
+ }
+
+ return false;
+ }
+
+ private bool TryParseCData(in SyntaxListBuilder builder)
+ {
+ if (CurrentToken.Kind == SyntaxKind.Text && string.Equals(CurrentToken.Content, "cdata", StringComparison.OrdinalIgnoreCase))
+ {
+ if (AcceptTokenAndMoveNext())
+ {
+ if (CurrentToken.Kind == SyntaxKind.LeftBracket)
+ {
+ return AcceptTokenUntilAll(builder, SyntaxKind.RightBracket, SyntaxKind.RightBracket, SyntaxKind.CloseAngle);
+ }
+ }
+ }
+
+ return false;
+ }
+
+ private bool TryParseXmlPI(in SyntaxListBuilder builder)
+ {
+ // Accept "?"
+ Assert(SyntaxKind.QuestionMark);
+ AcceptTokenAndMoveNext();
+ return AcceptTokenUntilAll(builder, SyntaxKind.QuestionMark, SyntaxKind.CloseAngle);
+ }
+
+ private void ParseOptionalBangEscape(in SyntaxListBuilder builder)
+ {
+ if (IsBangEscape(lookahead: 0))
+ {
+ builder.Add(OutputTokensAsHtmlLiteral());
+
+ // Accept the parser escape character '!'.
+ Assert(SyntaxKind.Bang);
+ AcceptTokenAndMoveNext();
+
+ // Setup the metacode span that we will be outputing.
+ SpanContext.ChunkGenerator = SpanChunkGenerator.Null;
+ builder.Add(OutputAsMetaCode(OutputTokens()));
+ }
+ }
+
+ private void SkipToEndScriptAndParseCode(in SyntaxListBuilder builder, AcceptedCharactersInternal endTagAcceptedCharacters = AcceptedCharactersInternal.Any)
+ {
+ // Special case for ] - SpanEditHandler;Accepts:Any - (27:0,27) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[script];
- SyntaxKind.CloseAngle;[>];
+RazorDocument - [0..36) - FullWidth: 36 - []
+ MarkupBlock - [0..36) - FullWidth: 36
+ MarkupTagBlock - [0..8) - FullWidth: 8
+ MarkupTextLiteral - [0..8) - FullWidth: 8 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ Text;[script];
+ CloseAngle;[>];
+ MarkupTextLiteral - [8..21) - FullWidth: 13 - Gen - SpanEditHandler;Accepts:Any
+ Text;[foo];
+ OpenAngle;[<];
+ Text;[bar];
+ Whitespace;[ ];
+ Text;[baz];
+ Equals;[=];
+ SingleQuote;['];
+ CSharpCodeBlock - [21..25) - FullWidth: 4
+ CSharpImplicitExpression - [21..25) - FullWidth: 4
+ CSharpTransition - [21..22) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ CSharpImplicitExpressionBody - [22..25) - FullWidth: 3
+ CSharpCodeBlock - [22..25) - FullWidth: 3
+ CSharpExpressionLiteral - [22..25) - FullWidth: 3 - Gen - ImplicitExpressionEditHandler;Accepts:NonWhitespace;ImplicitExpression[RTD];K14
+ Identifier;[boz];
+ MarkupTextLiteral - [25..27) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:Any
+ SingleQuote;['];
+ CloseAngle;[>];
+ MarkupTagBlock - [27..36) - FullWidth: 9
+ MarkupTextLiteral - [27..36) - FullWidth: 9 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[script];
+ CloseAngle;[>];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/NestedCodeBlockWithMarkupSetsDotAsMarkup.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/NestedCodeBlockWithMarkupSetsDotAsMarkup.stree.txt
index 4de831cbb..6450d7ca8 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/NestedCodeBlockWithMarkupSetsDotAsMarkup.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/NestedCodeBlockWithMarkupSetsDotAsMarkup.stree.txt
@@ -1,55 +1,59 @@
-Markup block - Gen - 52 - (0:0,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:1
- SyntaxKind.Unknown;[];
- Statement block - Gen - 52 - (0:0,0)
- Transition span - Gen - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
- SyntaxKind.Transition;[@];
- Code span - Gen - [if (true) { ] - SpanEditHandler;Accepts:Any - (1:0,1) - Tokens:8
- SyntaxKind.Keyword;[if];
- SyntaxKind.Whitespace;[ ];
- SyntaxKind.LeftParenthesis;[(];
- SyntaxKind.Keyword;[true];
- SyntaxKind.RightParenthesis;[)];
- SyntaxKind.Whitespace;[ ];
- SyntaxKind.LeftBrace;[{];
- SyntaxKind.Whitespace;[ ];
- Statement block - Gen - 37 - (13:0,13)
- Transition span - Gen - [@] - SpanEditHandler;Accepts:None - (13:0,13) - Tokens:1
- SyntaxKind.Transition;[@];
- Code span - Gen - [if(false) {] - SpanEditHandler;Accepts:Any - (14:0,14) - Tokens:6
- SyntaxKind.Keyword;[if];
- SyntaxKind.LeftParenthesis;[(];
- SyntaxKind.Keyword;[false];
- SyntaxKind.RightParenthesis;[)];
- SyntaxKind.Whitespace;[ ];
- SyntaxKind.LeftBrace;[{];
- Markup block - Gen - 24 - (25:0,25)
- Markup span - Gen - [ ] - SpanEditHandler;Accepts:Any - (25:0,25) - Tokens:1
- SyntaxKind.Whitespace;[ ];
- Tag block - Gen - 5 - (26:0,26)
- Markup span - Gen - [] - SpanEditHandler;Accepts:None - (26:0,26) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[div];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (31:0,31) - Tokens:1
- SyntaxKind.Unknown;[];
- Expression block - Gen - 10 - (31:0,31)
- Transition span - Gen - [@] - SpanEditHandler;Accepts:None - (31:0,31) - Tokens:1
- SyntaxKind.Transition;[@];
- Code span - Gen - [something] - ImplicitExpressionEditHandler;Accepts:NonWhitespace;ImplicitExpression[RTD];K14 - (32:0,32) - Tokens:1
- SyntaxKind.Identifier;[something];
- Markup span - Gen - [.] - SpanEditHandler;Accepts:Any - (41:0,41) - Tokens:1
- SyntaxKind.Text;[.];
- Tag block - Gen - 6 - (42:0,42)
- Markup span - Gen - [
] - SpanEditHandler;Accepts:None - (42:0,42) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[div];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [ ] - SpanEditHandler;Accepts:None - (48:0,48) - Tokens:1
- SyntaxKind.Whitespace;[ ];
- Code span - Gen - [}] - SpanEditHandler;Accepts:Any - (49:0,49) - Tokens:1
- SyntaxKind.RightBrace;[}];
- Code span - Gen - [ }] - SpanEditHandler;Accepts:Any - (50:0,50) - Tokens:2
- SyntaxKind.Whitespace;[ ];
- SyntaxKind.RightBrace;[}];
+RazorDocument - [0..52) - FullWidth: 52 - [@if (true) { @if(false) { @something.
} }]
+ MarkupBlock - [0..52) - FullWidth: 52
+ MarkupTextLiteral - [0..0) - FullWidth: 0 - Gen - SpanEditHandler;Accepts:Any
+ Marker;[];
+ CSharpCodeBlock - [0..52) - FullWidth: 52
+ CSharpTransition - [0..1) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ CSharpStatementLiteral - [1..13) - FullWidth: 12 - Gen - SpanEditHandler;Accepts:Any
+ Keyword;[if];
+ Whitespace;[ ];
+ LeftParenthesis;[(];
+ Keyword;[true];
+ RightParenthesis;[)];
+ Whitespace;[ ];
+ LeftBrace;[{];
+ Whitespace;[ ];
+ CSharpCodeBlock - [13..50) - FullWidth: 37
+ CSharpTransition - [13..14) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ CSharpStatementLiteral - [14..25) - FullWidth: 11 - Gen - SpanEditHandler;Accepts:Any
+ Keyword;[if];
+ LeftParenthesis;[(];
+ Keyword;[false];
+ RightParenthesis;[)];
+ Whitespace;[ ];
+ LeftBrace;[{];
+ MarkupBlock - [25..49) - FullWidth: 24
+ MarkupTextLiteral - [25..26) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:Any
+ Whitespace;[ ];
+ MarkupTagBlock - [26..31) - FullWidth: 5
+ MarkupTextLiteral - [26..31) - FullWidth: 5 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Text;[div];
+ CloseAngle;[>];
+ MarkupTextLiteral - [31..31) - FullWidth: 0 - Gen - SpanEditHandler;Accepts:Any
+ Marker;[];
+ CSharpCodeBlock - [31..41) - FullWidth: 10
+ CSharpImplicitExpression - [31..41) - FullWidth: 10
+ CSharpTransition - [31..32) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ CSharpImplicitExpressionBody - [32..41) - FullWidth: 9
+ CSharpCodeBlock - [32..41) - FullWidth: 9
+ CSharpExpressionLiteral - [32..41) - FullWidth: 9 - Gen - ImplicitExpressionEditHandler;Accepts:NonWhitespace;ImplicitExpression[RTD];K14
+ Identifier;[something];
+ MarkupTextLiteral - [41..42) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:Any
+ Text;[.];
+ MarkupTagBlock - [42..48) - FullWidth: 6
+ MarkupTextLiteral - [42..48) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[div];
+ CloseAngle;[>];
+ MarkupTextLiteral - [48..49) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Whitespace;[ ];
+ CSharpStatementLiteral - [49..50) - FullWidth: 1 - Gen - ImplicitExpressionEditHandler;Accepts:Any;ImplicitExpression[RTD];K14
+ RightBrace;[}];
+ CSharpStatementLiteral - [50..52) - FullWidth: 2 - Gen - ImplicitExpressionEditHandler;Accepts:Any;ImplicitExpression[ATD];K14
+ Whitespace;[ ];
+ RightBrace;[}];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/NoLongerSupportsDollarOpenBraceCombination.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/NoLongerSupportsDollarOpenBraceCombination.stree.txt
index f2a68845e..2923e610c 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/NoLongerSupportsDollarOpenBraceCombination.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/NoLongerSupportsDollarOpenBraceCombination.stree.txt
@@ -1,14 +1,15 @@
-Markup block - Gen - 17 - (0:0,0)
- Tag block - Gen - 5 - (0:0,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[foo];
- SyntaxKind.CloseAngle;[>];
- SyntaxKind.HtmlTextLiteral - [${bar}] - [5..11) - FullWidth: 6 - Slots: 1
- SyntaxKind.Text;[${bar}];
- Tag block - Gen - 6 - (11:0,11)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (11:0,11) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[foo];
- SyntaxKind.CloseAngle;[>];
+RazorDocument - [0..17) - FullWidth: 17 - [${bar}]
+ MarkupBlock - [0..17) - FullWidth: 17
+ MarkupTagBlock - [0..5) - FullWidth: 5
+ MarkupTextLiteral - [0..5) - FullWidth: 5 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ Text;[foo];
+ CloseAngle;[>];
+ MarkupTextLiteral - [5..11) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:Any
+ Text;[${bar}];
+ MarkupTagBlock - [11..17) - FullWidth: 6
+ MarkupTextLiteral - [11..17) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[foo];
+ CloseAngle;[>];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/OutputsEmptyBlockWithEmptyMarkupSpanIfContentIsEmptyString.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/OutputsEmptyBlockWithEmptyMarkupSpanIfContentIsEmptyString.stree.txt
index df48ed419..4e3000082 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/OutputsEmptyBlockWithEmptyMarkupSpanIfContentIsEmptyString.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/OutputsEmptyBlockWithEmptyMarkupSpanIfContentIsEmptyString.stree.txt
@@ -1,3 +1,4 @@
-Markup block - Gen - 0 - (0:0,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:1
- SyntaxKind.Unknown;[];
+RazorDocument - [0..0) - FullWidth: 0 - []
+ MarkupBlock - [0..0) - FullWidth: 0
+ MarkupTextLiteral - [0..0) - FullWidth: 0 - Gen - SpanEditHandler;Accepts:Any
+ Marker;[];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/OutputsWhitespaceOnlyContentAsSingleWhitespaceMarkupSpan.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/OutputsWhitespaceOnlyContentAsSingleWhitespaceMarkupSpan.stree.txt
index e22974936..f51e794b2 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/OutputsWhitespaceOnlyContentAsSingleWhitespaceMarkupSpan.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/OutputsWhitespaceOnlyContentAsSingleWhitespaceMarkupSpan.stree.txt
@@ -1,3 +1,4 @@
-Markup block - Gen - 10 - (0:0,0)
- Markup span - Gen - [ ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:1
- SyntaxKind.Whitespace;[ ];
+RazorDocument - [0..10) - FullWidth: 10 - [ ]
+ MarkupBlock - [0..10) - FullWidth: 10
+ MarkupTextLiteral - [0..10) - FullWidth: 10 - Gen - SpanEditHandler;Accepts:Any
+ Whitespace;[ ];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParseDocumentDoesNotSwitchToCodeOnEmailAddressInText.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParseDocumentDoesNotSwitchToCodeOnEmailAddressInText.stree.txt
index 6c9f0060a..b40a8668c 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParseDocumentDoesNotSwitchToCodeOnEmailAddressInText.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParseDocumentDoesNotSwitchToCodeOnEmailAddressInText.stree.txt
@@ -1,3 +1,4 @@
-Markup block - Gen - 21 - (0:0,0)
- Markup span - Gen - [example@microsoft.com] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:1
- SyntaxKind.Text;[example@microsoft.com];
+RazorDocument - [0..21) - FullWidth: 21 - [example@microsoft.com]
+ MarkupBlock - [0..21) - FullWidth: 21
+ MarkupTextLiteral - [0..21) - FullWidth: 21 - Gen - SpanEditHandler;Accepts:Any
+ Text;[example@microsoft.com];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParseSectionIgnoresTagsInContentsOfScriptTag.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParseSectionIgnoresTagsInContentsOfScriptTag.stree.txt
index 9b49f113c..255197a72 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParseSectionIgnoresTagsInContentsOfScriptTag.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParseSectionIgnoresTagsInContentsOfScriptTag.stree.txt
@@ -1,52 +1,59 @@
-Markup block - Gen - 53 - (0:0,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:1
- SyntaxKind.Unknown;[];
- Directive block - Gen - 53 - (0:0,0)
- Transition span - Gen - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
- SyntaxKind.Transition;[@];
- MetaCode span - Gen - [section] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
- SyntaxKind.Identifier;[section];
- Code span - Gen - [ ] - SpanEditHandler;Accepts:Whitespace - (8:0,8) - Tokens:1
- SyntaxKind.Whitespace;[ ];
- Code span - Gen - [Foo] - DirectiveTokenEditHandler;Accepts:NonWhitespace - (9:0,9) - Tokens:1
- SyntaxKind.Identifier;[Foo];
- Markup span - Gen - [ ] - SpanEditHandler;Accepts:AllWhitespace - (12:0,12) - Tokens:1
- SyntaxKind.Whitespace;[ ];
- MetaCode span - Gen - [{] - AutoCompleteEditHandler;Accepts:None,AutoComplete:[];AtEnd - (13:0,13) - Tokens:1
- SyntaxKind.LeftBrace;[{];
- Markup block - Gen - 38 - (14:0,14)
- Markup span - Gen - [ ] - SpanEditHandler;Accepts:Any - (14:0,14) - Tokens:1
- SyntaxKind.Whitespace;[ ];
- Tag block - Gen - 8 - (15:0,15)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (42:0,42) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[script];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [ ] - SpanEditHandler;Accepts:Any - (51:0,51) - Tokens:1
- SyntaxKind.Whitespace;[ ];
- MetaCode span - Gen - [}] - SpanEditHandler;Accepts:None - (52:0,52) - Tokens:1
- SyntaxKind.RightBrace;[}];
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (53:0,53) - Tokens:1
- SyntaxKind.Unknown;[];
+RazorDocument - [0..53) - FullWidth: 53 - [@section Foo { }]
+ MarkupBlock - [0..53) - FullWidth: 53
+ MarkupTextLiteral - [0..0) - FullWidth: 0 - Gen - SpanEditHandler;Accepts:Any
+ Marker;[];
+ CSharpCodeBlock - [0..53) - FullWidth: 53
+ RazorDirective - [0..53) - FullWidth: 53
+ CSharpTransition - [0..1) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ RazorDirectiveBody - [1..53) - FullWidth: 52
+ RazorMetaCode - [1..8) - FullWidth: 7 - Gen - SpanEditHandler;Accepts:None
+ Identifier;[section];
+ CSharpCodeBlock - [8..53) - FullWidth: 45
+ CSharpStatementLiteral - [8..9) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:Whitespace
+ Whitespace;[ ];
+ CSharpStatementLiteral - [9..12) - FullWidth: 3 - Gen - DirectiveTokenEditHandler;Accepts:NonWhitespace
+ Identifier;[Foo];
+ MarkupTextLiteral - [12..13) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:AllWhitespace
+ Whitespace;[ ];
+ RazorMetaCode - [13..14) - FullWidth: 1 - Gen - AutoCompleteEditHandler;Accepts:None,AutoComplete:[];AtEnd
+ LeftBrace;[{];
+ MarkupBlock - [14..52) - FullWidth: 38
+ MarkupTextLiteral - [14..15) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:Any
+ Whitespace;[ ];
+ MarkupTagBlock - [15..23) - FullWidth: 8
+ MarkupTextLiteral - [15..23) - FullWidth: 8 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ Text;[script];
+ CloseAngle;[>];
+ MarkupTextLiteral - [23..36) - FullWidth: 13 - Gen - SpanEditHandler;Accepts:Any
+ Text;[foo];
+ OpenAngle;[<];
+ Text;[bar];
+ Whitespace;[ ];
+ Text;[baz];
+ Equals;[=];
+ SingleQuote;['];
+ CSharpCodeBlock - [36..40) - FullWidth: 4
+ CSharpImplicitExpression - [36..40) - FullWidth: 4
+ CSharpTransition - [36..37) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ CSharpImplicitExpressionBody - [37..40) - FullWidth: 3
+ CSharpCodeBlock - [37..40) - FullWidth: 3
+ CSharpExpressionLiteral - [37..40) - FullWidth: 3 - Gen - ImplicitExpressionEditHandler;Accepts:NonWhitespace;ImplicitExpression[RTD];K15
+ Identifier;[boz];
+ MarkupTextLiteral - [40..42) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:Any
+ SingleQuote;['];
+ CloseAngle;[>];
+ MarkupTagBlock - [42..51) - FullWidth: 9
+ MarkupTextLiteral - [42..51) - FullWidth: 9 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[script];
+ CloseAngle;[>];
+ MarkupTextLiteral - [51..52) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:Any
+ Whitespace;[ ];
+ RazorMetaCode - [52..53) - FullWidth: 1 - Gen - ImplicitExpressionEditHandler;Accepts:None;ImplicitExpression[RTD];K15
+ RightBrace;[}];
+ MarkupTextLiteral - [53..53) - FullWidth: 0 - Gen - SpanEditHandler;Accepts:Any
+ Marker;[];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParsesWholeContentAsOneSpanIfNoSwapCharacterEncountered.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParsesWholeContentAsOneSpanIfNoSwapCharacterEncountered.stree.txt
index 5cf09d83c..b8fbb1060 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParsesWholeContentAsOneSpanIfNoSwapCharacterEncountered.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ParsesWholeContentAsOneSpanIfNoSwapCharacterEncountered.stree.txt
@@ -1,5 +1,6 @@
-Markup block - Gen - 7 - (0:0,0)
- Markup span - Gen - [foo baz] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:3
- SyntaxKind.Text;[foo];
- SyntaxKind.Whitespace;[ ];
- SyntaxKind.Text;[baz];
+RazorDocument - [0..7) - FullWidth: 7 - [foo baz]
+ MarkupBlock - [0..7) - FullWidth: 7
+ MarkupTextLiteral - [0..7) - FullWidth: 7 - Gen - SpanEditHandler;Accepts:Any
+ Text;[foo];
+ Whitespace;[ ];
+ Text;[baz];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersExtraNewlineAtTheEndTextTagInVerbatimBlockIfFollowedByHtml.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersExtraNewlineAtTheEndTextTagInVerbatimBlockIfFollowedByHtml.stree.txt
index 16d62d841..3f6aa2076 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersExtraNewlineAtTheEndTextTagInVerbatimBlockIfFollowedByHtml.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersExtraNewlineAtTheEndTextTagInVerbatimBlockIfFollowedByHtml.stree.txt
@@ -1,42 +1,46 @@
-Markup block - Gen - 38 - (0:0,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:1
- SyntaxKind.Unknown;[];
- Statement block - Gen - 32 - (0:0,0)
- Transition span - Gen - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
- SyntaxKind.Transition;[@];
- MetaCode span - Gen - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
- SyntaxKind.LeftBrace;[{];
- Markup block - Gen - 19 - (2:0,2)
- Tag block - Gen - 6 - (2:0,2)
- Transition span - Gen - [] - SpanEditHandler;Accepts:None - (2:0,2) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[text];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [Blah] - SpanEditHandler;Accepts:None - (8:0,8) - Tokens:1
- SyntaxKind.Text;[Blah];
- Tag block - Gen - 7 - (12:0,12)
- Transition span - Gen - [] - SpanEditHandler;Accepts:None - (12:0,12) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[text];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [LF] - SpanEditHandler;Accepts:None - (19:0,19) - Tokens:1
- SyntaxKind.NewLine;[LF];
- Markup block - Gen - 10 - (21:1,0)
- Tag block - Gen - 8 - (21:1,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:None - (21:1,0) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[input];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [LF] - SpanEditHandler;Accepts:None - (29:1,8) - Tokens:1
- SyntaxKind.NewLine;[LF];
- Code span - Gen - [] - SpanEditHandler;Accepts:Any - (31:2,0) - Tokens:1
- SyntaxKind.Unknown;[];
- MetaCode span - Gen - [}] - SpanEditHandler;Accepts:None - (31:2,0) - Tokens:1
- SyntaxKind.RightBrace;[}];
- Tag block - Gen - 6 - (32:2,1)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (32:2,1) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[html];
- SyntaxKind.CloseAngle;[>];
+RazorDocument - [0..38) - FullWidth: 38 - [@{BlahLFLF}]
+ MarkupBlock - [0..38) - FullWidth: 38
+ MarkupTextLiteral - [0..0) - FullWidth: 0 - Gen - SpanEditHandler;Accepts:Any
+ Marker;[];
+ CSharpCodeBlock - [0..32) - FullWidth: 32
+ CSharpStatement - [0..32) - FullWidth: 32
+ CSharpTransition - [0..1) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ CSharpStatementBody - [1..32) - FullWidth: 31
+ RazorMetaCode - [1..2) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ LeftBrace;[{];
+ CSharpCodeBlock - [2..31) - FullWidth: 29
+ MarkupBlock - [2..21) - FullWidth: 19
+ MarkupTagBlock - [2..8) - FullWidth: 6
+ MarkupTransition - [2..8) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Text;[text];
+ CloseAngle;[>];
+ MarkupTextLiteral - [8..12) - FullWidth: 4 - Gen - SpanEditHandler;Accepts:None
+ Text;[Blah];
+ MarkupTagBlock - [12..19) - FullWidth: 7
+ MarkupTransition - [12..19) - FullWidth: 7 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[text];
+ CloseAngle;[>];
+ MarkupTextLiteral - [19..21) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:None
+ NewLine;[LF];
+ MarkupBlock - [21..31) - FullWidth: 10
+ MarkupTagBlock - [21..29) - FullWidth: 8
+ MarkupTextLiteral - [21..29) - FullWidth: 8 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Text;[input];
+ ForwardSlash;[/];
+ CloseAngle;[>];
+ MarkupTextLiteral - [29..31) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:None
+ NewLine;[LF];
+ CSharpStatementLiteral - [31..31) - FullWidth: 0 - Gen - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[];AtEOL
+ Marker;[];
+ RazorMetaCode - [31..32) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ RightBrace;[}];
+ MarkupTagBlock - [32..38) - FullWidth: 6
+ MarkupTextLiteral - [32..38) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ Text;[html];
+ CloseAngle;[>];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersNewlineAfterTextTagInVerbatimBlockIfFollowedByMarkupTransition.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersNewlineAfterTextTagInVerbatimBlockIfFollowedByMarkupTransition.stree.txt
index e0d845243..eb8bd65f2 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersNewlineAfterTextTagInVerbatimBlockIfFollowedByMarkupTransition.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersNewlineAfterTextTagInVerbatimBlockIfFollowedByMarkupTransition.stree.txt
@@ -1,42 +1,46 @@
-Markup block - Gen - 37 - (0:0,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:1
- SyntaxKind.Unknown;[];
- Statement block - Gen - 31 - (0:0,0)
- Transition span - Gen - [@] - SpanEditHandler;Accepts:None - (0:0,0) - Tokens:1
- SyntaxKind.Transition;[@];
- MetaCode span - Gen - [{] - SpanEditHandler;Accepts:None - (1:0,1) - Tokens:1
- SyntaxKind.LeftBrace;[{];
- Markup block - Gen - 19 - (2:0,2)
- Tag block - Gen - 6 - (2:0,2)
- Transition span - Gen - [] - SpanEditHandler;Accepts:None - (2:0,2) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[text];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [Blah] - SpanEditHandler;Accepts:None - (8:0,8) - Tokens:1
- SyntaxKind.Text;[Blah];
- Tag block - Gen - 7 - (12:0,12)
- Transition span - Gen - [] - SpanEditHandler;Accepts:None - (12:0,12) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[text];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [LF] - SpanEditHandler;Accepts:None - (19:0,19) - Tokens:1
- SyntaxKind.NewLine;[LF];
- Markup block - Gen - 9 - (21:1,0)
- Transition span - Gen - [@] - SpanEditHandler;Accepts:None - (21:1,0) - Tokens:1
- SyntaxKind.Transition;[@];
- MetaCode span - Gen - [:] - SpanEditHandler;Accepts:Any - (22:1,1) - Tokens:1
- SyntaxKind.Colon;[:];
- Markup span - Gen - [ BlehLF] - SpanEditHandler;Accepts:None - (23:1,2) - Tokens:3
- SyntaxKind.Whitespace;[ ];
- SyntaxKind.Text;[Bleh];
- SyntaxKind.NewLine;[LF];
- Code span - Gen - [] - SpanEditHandler;Accepts:Any - (30:2,0) - Tokens:1
- SyntaxKind.Unknown;[];
- MetaCode span - Gen - [}] - SpanEditHandler;Accepts:None - (30:2,0) - Tokens:1
- SyntaxKind.RightBrace;[}];
- Tag block - Gen - 6 - (31:2,1)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (31:2,1) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[html];
- SyntaxKind.CloseAngle;[>];
+RazorDocument - [0..37) - FullWidth: 37 - [@{BlahLF@: BlehLF}]
+ MarkupBlock - [0..37) - FullWidth: 37
+ MarkupTextLiteral - [0..0) - FullWidth: 0 - Gen - SpanEditHandler;Accepts:Any
+ Marker;[];
+ CSharpCodeBlock - [0..31) - FullWidth: 31
+ CSharpStatement - [0..31) - FullWidth: 31
+ CSharpTransition - [0..1) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ CSharpStatementBody - [1..31) - FullWidth: 30
+ RazorMetaCode - [1..2) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ LeftBrace;[{];
+ CSharpCodeBlock - [2..30) - FullWidth: 28
+ MarkupBlock - [2..21) - FullWidth: 19
+ MarkupTagBlock - [2..8) - FullWidth: 6
+ MarkupTransition - [2..8) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Text;[text];
+ CloseAngle;[>];
+ MarkupTextLiteral - [8..12) - FullWidth: 4 - Gen - SpanEditHandler;Accepts:None
+ Text;[Blah];
+ MarkupTagBlock - [12..19) - FullWidth: 7
+ MarkupTransition - [12..19) - FullWidth: 7 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[text];
+ CloseAngle;[>];
+ MarkupTextLiteral - [19..21) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:None
+ NewLine;[LF];
+ MarkupBlock - [21..30) - FullWidth: 9
+ MarkupTransition - [21..22) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ RazorMetaCode - [22..23) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:Any
+ Colon;[:];
+ MarkupTextLiteral - [23..30) - FullWidth: 7 - Gen - SpanEditHandler;Accepts:None
+ Whitespace;[ ];
+ Text;[Bleh];
+ NewLine;[LF];
+ CSharpStatementLiteral - [30..30) - FullWidth: 0 - Gen - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[];AtEOL
+ Marker;[];
+ RazorMetaCode - [30..31) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ RightBrace;[}];
+ MarkupTagBlock - [31..37) - FullWidth: 6
+ MarkupTextLiteral - [31..37) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ Text;[html];
+ CloseAngle;[>];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersTextPseudoTagAsMarkup.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersTextPseudoTagAsMarkup.stree.txt
index 3e301a21b..77e5a0492 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersTextPseudoTagAsMarkup.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/RendersTextPseudoTagAsMarkup.stree.txt
@@ -1,17 +1,18 @@
-Markup block - Gen - 20 - (0:0,0)
- Markup span - Gen - [Foo ] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:2
- SyntaxKind.Text;[Foo];
- SyntaxKind.Whitespace;[ ];
- Tag block - Gen - 6 - (4:0,4)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (4:0,4) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[text];
- SyntaxKind.CloseAngle;[>];
- SyntaxKind.HtmlTextLiteral - [Foo] - [10..13) - FullWidth: 3 - Slots: 1
- SyntaxKind.Text;[Foo];
- Tag block - Gen - 7 - (13:0,13)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (13:0,13) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[text];
- SyntaxKind.CloseAngle;[>];
+RazorDocument - [0..20) - FullWidth: 20 - [Foo Foo]
+ MarkupBlock - [0..20) - FullWidth: 20
+ MarkupTextLiteral - [0..4) - FullWidth: 4 - Gen - SpanEditHandler;Accepts:Any
+ Text;[Foo];
+ Whitespace;[ ];
+ MarkupTagBlock - [4..10) - FullWidth: 6
+ MarkupTextLiteral - [4..10) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ Text;[text];
+ CloseAngle;[>];
+ MarkupTextLiteral - [10..13) - FullWidth: 3 - Gen - SpanEditHandler;Accepts:Any
+ Text;[Foo];
+ MarkupTagBlock - [13..20) - FullWidth: 7
+ MarkupTextLiteral - [13..20) - FullWidth: 7 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[text];
+ CloseAngle;[>];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ReturnsOneMarkupSegmentIfNoCodeBlocksEncountered.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ReturnsOneMarkupSegmentIfNoCodeBlocksEncountered.stree.txt
index 20d868ac4..53ac9fc97 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ReturnsOneMarkupSegmentIfNoCodeBlocksEncountered.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlDocumentTest/ReturnsOneMarkupSegmentIfNoCodeBlocksEncountered.stree.txt
@@ -1,25 +1,26 @@
-Markup block - Gen - 30 - (0:0,0)
- Markup span - Gen - [Foo Baz] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:3
- SyntaxKind.Text;[Foo];
- SyntaxKind.Whitespace;[ ];
- SyntaxKind.Text;[Baz];
- HtmlComment block - Gen - 10 - (7:0,7)
- Markup span - Gen - [] - SpanEditHandler;Accepts:None - (14:0,14) - Tokens:2
- SyntaxKind.DoubleHyphen;[--];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [Bar] - SpanEditHandler;Accepts:Any - (17:0,17) - Tokens:1
- SyntaxKind.Text;[Bar];
- Markup span - Gen - [Bar] - SpanEditHandler;Accepts:None - (62:1,58) - Tokens:2
- SyntaxKind.DoubleHyphen;[--];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [LF] - SpanEditHandler;Accepts:None - (65:1,61) - Tokens:1
- SyntaxKind.NewLine;[LF];
- Code span - Gen - [] - SpanEditHandler;Accepts:Any - (67:2,0) - Tokens:1
- SyntaxKind.Unknown;[];
- MetaCode span - Gen - [}] - SpanEditHandler;Accepts:None - (67:2,0) - Tokens:1
- SyntaxKind.RightBrace;[}];
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (68:2,1) - Tokens:1
- SyntaxKind.Unknown;[];
+RazorDocument - [0..68) - FullWidth: 68 - [@{LF LF}]
+ MarkupBlock - [0..68) - FullWidth: 68
+ MarkupTextLiteral - [0..0) - FullWidth: 0 - Gen - SpanEditHandler;Accepts:Any
+ Marker;[];
+ CSharpCodeBlock - [0..68) - FullWidth: 68
+ CSharpStatement - [0..68) - FullWidth: 68
+ CSharpTransition - [0..1) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ CSharpStatementBody - [1..68) - FullWidth: 67
+ RazorMetaCode - [1..2) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ LeftBrace;[{];
+ CSharpCodeBlock - [2..67) - FullWidth: 65
+ CSharpStatementLiteral - [2..4) - FullWidth: 2 - Gen - AutoCompleteEditHandler;Accepts:Any,AutoComplete:[];AtEOL
+ NewLine;[LF];
+ MarkupBlock - [4..67) - FullWidth: 63
+ MarkupTextLiteral - [4..8) - FullWidth: 4 - Gen - SpanEditHandler;Accepts:Any
+ Whitespace;[ ];
+ MarkupCommentBlock - [8..65) - FullWidth: 57
+ MarkupTextLiteral - [8..12) - FullWidth: 4 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Bang;[!];
+ DoubleHyphen;[--];
+ MarkupTextLiteral - [12..62) - FullWidth: 50 - Gen - SpanEditHandler;Accepts:Whitespace
+ Whitespace;[ ];
+ Text;[Hello,];
+ Whitespace;[ ];
+ Text;[I];
+ SingleQuote;['];
+ Text;[m];
+ Whitespace;[ ];
+ Text;[a];
+ Whitespace;[ ];
+ Text;[comment];
+ Whitespace;[ ];
+ Text;[that];
+ Whitespace;[ ];
+ Text;[shouldn];
+ SingleQuote;['];
+ Text;[t];
+ Whitespace;[ ];
+ Text;[break];
+ Whitespace;[ ];
+ Text;[razor];
+ Whitespace;[ ];
+ Text;[-];
+ MarkupTextLiteral - [62..65) - FullWidth: 3 - Gen - SpanEditHandler;Accepts:None
+ DoubleHyphen;[--];
+ CloseAngle;[>];
+ MarkupTextLiteral - [65..67) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:None
+ NewLine;[LF];
+ CSharpStatementLiteral - [67..67) - FullWidth: 0 - Gen - SpanEditHandler;Accepts:Any
+ Marker;[];
+ RazorMetaCode - [67..68) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ RightBrace;[}];
+ MarkupTextLiteral - [68..68) - FullWidth: 0 - Gen - SpanEditHandler;Accepts:Any
+ Marker;[];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/HtmlCommentSupportsMultipleDashes.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/HtmlCommentSupportsMultipleDashes.stree.txt
index 3679bca5b..e4bd554c9 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/HtmlCommentSupportsMultipleDashes.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/HtmlCommentSupportsMultipleDashes.stree.txt
@@ -1,133 +1,134 @@
-Markup block - Gen - 165 - (0:0,0)
- Tag block - Gen - 5 - (0:0,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (0:0,0) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[div];
- SyntaxKind.CloseAngle;[>];
- HtmlComment block - Gen - 22 - (5:0,5)
- Markup span - Gen - [] - SpanEditHandler;Accepts:None - (24:0,24) - Tokens:2
- SyntaxKind.DoubleHyphen;[--];
- SyntaxKind.CloseAngle;[>];
- Tag block - Gen - 6 - (27:0,27)
- Markup span - Gen - [
] - SpanEditHandler;Accepts:Any - (27:0,27) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[div];
- SyntaxKind.CloseAngle;[>];
- SyntaxKind.HtmlTextLiteral - [LF] - [33..35) - FullWidth: 2 - Slots: 1
- SyntaxKind.NewLine;[LF];
- Tag block - Gen - 5 - (35:1,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (35:1,0) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[div];
- SyntaxKind.CloseAngle;[>];
- HtmlComment block - Gen - 24 - (40:1,5)
- Markup span - Gen - [] - SpanEditHandler;Accepts:None - (61:1,26) - Tokens:2
- SyntaxKind.DoubleHyphen;[--];
- SyntaxKind.CloseAngle;[>];
- Tag block - Gen - 6 - (64:1,29)
- Markup span - Gen - [
] - SpanEditHandler;Accepts:Any - (64:1,29) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[div];
- SyntaxKind.CloseAngle;[>];
- SyntaxKind.HtmlTextLiteral - [LF] - [70..72) - FullWidth: 2 - Slots: 1
- SyntaxKind.NewLine;[LF];
- Tag block - Gen - 5 - (72:2,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (72:2,0) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[div];
- SyntaxKind.CloseAngle;[>];
- HtmlComment block - Gen - 26 - (77:2,5)
- Markup span - Gen - [] - SpanEditHandler;Accepts:None - (100:2,28) - Tokens:2
- SyntaxKind.DoubleHyphen;[--];
- SyntaxKind.CloseAngle;[>];
- Tag block - Gen - 6 - (103:2,31)
- Markup span - Gen - [
] - SpanEditHandler;Accepts:Any - (103:2,31) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[div];
- SyntaxKind.CloseAngle;[>];
- SyntaxKind.HtmlTextLiteral - [LF] - [109..111) - FullWidth: 2 - Slots: 1
- SyntaxKind.NewLine;[LF];
- Tag block - Gen - 5 - (111:3,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:Any - (111:3,0) - Tokens:3
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.Text;[div];
- SyntaxKind.CloseAngle;[>];
- HtmlComment block - Gen - 41 - (116:3,5)
- Markup span - Gen - [] - SpanEditHandler;Accepts:None - (154:3,43) - Tokens:2
- SyntaxKind.DoubleHyphen;[--];
- SyntaxKind.CloseAngle;[>];
- Tag block - Gen - 6 - (157:3,46)
- Markup span - Gen - [
] - SpanEditHandler;Accepts:Any - (157:3,46) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[div];
- SyntaxKind.CloseAngle;[>];
- Markup span - Gen - [LF] - SpanEditHandler;Accepts:Any - (163:3,52) - Tokens:1
- SyntaxKind.NewLine;[LF];
+RazorDocument - [0..165) - FullWidth: 165 - [LFLFLFLF]
+ MarkupBlock - [0..165) - FullWidth: 165
+ MarkupTagBlock - [0..5) - FullWidth: 5
+ MarkupTextLiteral - [0..5) - FullWidth: 5 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ Text;[div];
+ CloseAngle;[>];
+ MarkupCommentBlock - [5..27) - FullWidth: 22
+ MarkupTextLiteral - [5..9) - FullWidth: 4 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Bang;[!];
+ DoubleHyphen;[--];
+ MarkupTextLiteral - [9..24) - FullWidth: 15 - Gen - SpanEditHandler;Accepts:Whitespace
+ Text;[-];
+ Whitespace;[ ];
+ Text;[Hello];
+ Whitespace;[ ];
+ Text;[World];
+ Whitespace;[ ];
+ Text;[-];
+ MarkupTextLiteral - [24..27) - FullWidth: 3 - Gen - SpanEditHandler;Accepts:None
+ DoubleHyphen;[--];
+ CloseAngle;[>];
+ MarkupTagBlock - [27..33) - FullWidth: 6
+ MarkupTextLiteral - [27..33) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[div];
+ CloseAngle;[>];
+ MarkupTextLiteral - [33..35) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:Any
+ NewLine;[LF];
+ MarkupTagBlock - [35..40) - FullWidth: 5
+ MarkupTextLiteral - [35..40) - FullWidth: 5 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ Text;[div];
+ CloseAngle;[>];
+ MarkupCommentBlock - [40..64) - FullWidth: 24
+ MarkupTextLiteral - [40..44) - FullWidth: 4 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Bang;[!];
+ DoubleHyphen;[--];
+ MarkupTextLiteral - [44..61) - FullWidth: 17 - Gen - SpanEditHandler;Accepts:Whitespace
+ DoubleHyphen;[--];
+ Whitespace;[ ];
+ Text;[Hello];
+ Whitespace;[ ];
+ Text;[World];
+ Whitespace;[ ];
+ DoubleHyphen;[--];
+ MarkupTextLiteral - [61..64) - FullWidth: 3 - Gen - SpanEditHandler;Accepts:None
+ DoubleHyphen;[--];
+ CloseAngle;[>];
+ MarkupTagBlock - [64..70) - FullWidth: 6
+ MarkupTextLiteral - [64..70) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[div];
+ CloseAngle;[>];
+ MarkupTextLiteral - [70..72) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:Any
+ NewLine;[LF];
+ MarkupTagBlock - [72..77) - FullWidth: 5
+ MarkupTextLiteral - [72..77) - FullWidth: 5 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ Text;[div];
+ CloseAngle;[>];
+ MarkupCommentBlock - [77..103) - FullWidth: 26
+ MarkupTextLiteral - [77..81) - FullWidth: 4 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Bang;[!];
+ DoubleHyphen;[--];
+ MarkupTextLiteral - [81..100) - FullWidth: 19 - Gen - SpanEditHandler;Accepts:Whitespace
+ DoubleHyphen;[--];
+ Text;[-];
+ Whitespace;[ ];
+ Text;[Hello];
+ Whitespace;[ ];
+ Text;[World];
+ Whitespace;[ ];
+ DoubleHyphen;[--];
+ Text;[-];
+ MarkupTextLiteral - [100..103) - FullWidth: 3 - Gen - SpanEditHandler;Accepts:None
+ DoubleHyphen;[--];
+ CloseAngle;[>];
+ MarkupTagBlock - [103..109) - FullWidth: 6
+ MarkupTextLiteral - [103..109) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[div];
+ CloseAngle;[>];
+ MarkupTextLiteral - [109..111) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:Any
+ NewLine;[LF];
+ MarkupTagBlock - [111..116) - FullWidth: 5
+ MarkupTextLiteral - [111..116) - FullWidth: 5 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ Text;[div];
+ CloseAngle;[>];
+ MarkupCommentBlock - [116..157) - FullWidth: 41
+ MarkupTextLiteral - [116..120) - FullWidth: 4 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Bang;[!];
+ DoubleHyphen;[--];
+ MarkupTextLiteral - [120..154) - FullWidth: 34 - Gen - SpanEditHandler;Accepts:Whitespace
+ DoubleHyphen;[--];
+ Text;[-];
+ Whitespace;[ ];
+ Text;[Hello];
+ Whitespace;[ ];
+ OpenAngle;[<];
+ Whitespace;[ ];
+ DoubleHyphen;[--];
+ Text;[-];
+ Whitespace;[ ];
+ CloseAngle;[>];
+ Whitespace;[ ];
+ Text;[World];
+ Whitespace;[ ];
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[div];
+ CloseAngle;[>];
+ Whitespace;[ ];
+ DoubleHyphen;[--];
+ Text;[-];
+ MarkupTextLiteral - [154..157) - FullWidth: 3 - Gen - SpanEditHandler;Accepts:None
+ DoubleHyphen;[--];
+ CloseAngle;[>];
+ MarkupTagBlock - [157..163) - FullWidth: 6
+ MarkupTextLiteral - [157..163) - FullWidth: 6 - Gen - SpanEditHandler;Accepts:Any
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[div];
+ CloseAngle;[>];
+ MarkupTextLiteral - [163..165) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:Any
+ NewLine;[LF];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/IgnoresTagsInContentsOfScriptTag.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/IgnoresTagsInContentsOfScriptTag.stree.txt
index d60e3e493..98eb40f52 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/IgnoresTagsInContentsOfScriptTag.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/IgnoresTagsInContentsOfScriptTag.stree.txt
@@ -1,28 +1,31 @@
-Markup block - Gen - 36 - (0:0,0)
- Tag block - Gen - 8 - (0:0,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:None - (27:0,27) - Tokens:4
- SyntaxKind.OpenAngle;[<];
- SyntaxKind.ForwardSlash;[/];
- SyntaxKind.Text;[script];
- SyntaxKind.CloseAngle;[>];
+MarkupBlock - [0..36) - FullWidth: 36 - []
+ MarkupTagBlock - [0..8) - FullWidth: 8
+ MarkupTextLiteral - [0..8) - FullWidth: 8 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Text;[script];
+ CloseAngle;[>];
+ MarkupTextLiteral - [8..21) - FullWidth: 13 - Gen - SpanEditHandler;Accepts:Any
+ Text;[foo];
+ OpenAngle;[<];
+ Text;[bar];
+ Whitespace;[ ];
+ Text;[baz];
+ Equals;[=];
+ SingleQuote;['];
+ CSharpCodeBlock - [21..25) - FullWidth: 4
+ CSharpImplicitExpression - [21..25) - FullWidth: 4
+ CSharpTransition - [21..22) - FullWidth: 1 - Gen - SpanEditHandler;Accepts:None
+ Transition;[@];
+ CSharpImplicitExpressionBody - [22..25) - FullWidth: 3
+ CSharpCodeBlock - [22..25) - FullWidth: 3
+ CSharpExpressionLiteral - [22..25) - FullWidth: 3 - Gen - ImplicitExpressionEditHandler;Accepts:NonWhitespace;ImplicitExpression[RTD];K14
+ Identifier;[boz];
+ MarkupTextLiteral - [25..27) - FullWidth: 2 - Gen - SpanEditHandler;Accepts:Any
+ SingleQuote;['];
+ CloseAngle;[>];
+ MarkupTagBlock - [27..36) - FullWidth: 9
+ MarkupTextLiteral - [27..36) - FullWidth: 9 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[script];
+ CloseAngle;[>];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/OnlyTerminatesCommentOnFullEndSequence.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/OnlyTerminatesCommentOnFullEndSequence.stree.txt
index 22d402b96..bddb0fb71 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/OnlyTerminatesCommentOnFullEndSequence.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/OnlyTerminatesCommentOnFullEndSequence.stree.txt
@@ -1,18 +1,18 @@
-Markup block - Gen - 20 - (0:0,0)
- HtmlComment block - Gen - 20 - (0:0,0)
- Markup span - Gen - [] - SpanEditHandler;Accepts:None - (17:0,17) - Tokens:2
- SyntaxKind.DoubleHyphen;[--];
- SyntaxKind.CloseAngle;[>];
+MarkupBlock - [0..20) - FullWidth: 20 - []
+ MarkupCommentBlock - [0..20) - FullWidth: 20
+ MarkupTextLiteral - [0..4) - FullWidth: 4 - Gen - SpanEditHandler;Accepts:None
+ OpenAngle;[<];
+ Bang;[!];
+ DoubleHyphen;[--];
+ MarkupTextLiteral - [4..17) - FullWidth: 13 - Gen - SpanEditHandler;Accepts:Whitespace
+ OpenAngle;[<];
+ Text;[foo];
+ CloseAngle;[>];
+ DoubleHyphen;[--];
+ OpenAngle;[<];
+ ForwardSlash;[/];
+ Text;[bar];
+ CloseAngle;[>];
+ MarkupTextLiteral - [17..20) - FullWidth: 3 - Gen - SpanEditHandler;Accepts:None
+ DoubleHyphen;[--];
+ CloseAngle;[>];
diff --git a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/ParsesSGMLDeclarationAsEmptyTag.stree.txt b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/ParsesSGMLDeclarationAsEmptyTag.stree.txt
index b3b77f162..cbf3ab2bd 100644
--- a/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/ParsesSGMLDeclarationAsEmptyTag.stree.txt
+++ b/test/Microsoft.AspNetCore.Razor.Language.Test/TestFiles/ParserTests/HtmlBlockTest/ParsesSGMLDeclarationAsEmptyTag.stree.txt
@@ -1,23 +1,23 @@
-Markup block - Gen