Bring back SourceFile.EndOfFileToken

CHANGES.md

@@ -6,15 +6,14 @@ CHANGES.md lists intentional changes between the Strada (Typescript) and Corsa (
 
 ## Parser
 
-1. Source files do not contain an EndOfFile token as their last child.
-2. Malformed `...T?` at the end of a tuple now fails with a parse error instead of a grammar error.
-3. Malformed string ImportSpecifiers (`import x as "OOPS" from "y"`) now contain the string's text instead of an empty identifier.
-4. Empty binding elements no longer have a separate kind for OmittedExpression. Instead they have Kind=BindingElement with a nil Initialiser, Name and DotDotDotToken.
-5. ShorthandPropertyAssignment no longer includes an EqualsToken as a child when it has an ObjectAssignmentInitializer.
-6. JSDoc nodes now include leading whitespace in their location.
-7. The parser always parses a JSDocText node for comments in JSDoc. `string` is no longer part of the type of `comment`.
-8. In cases where Strada did produce a JSDocText node, Corsa no longer (incorrectly) includes all leading and trailing whitespace/asterisks, as well as initial `/**`.
-9. JSDocMemberName is now parsed as QualifiedName. These two nodes previously only differed by type, and now QualifiedName has a much less restrictive type for its left child.
+1. Malformed `...T?` at the end of a tuple now fails with a parse error instead of a grammar error.
+2. Malformed string ImportSpecifiers (`import x as "OOPS" from "y"`) now contain the string's text instead of an empty identifier.
+3. Empty binding elements no longer have a separate kind for OmittedExpression. Instead they have Kind=BindingElement with a nil Initialiser, Name and DotDotDotToken.
+4. ShorthandPropertyAssignment no longer includes an EqualsToken as a child when it has an ObjectAssignmentInitializer.
+5. JSDoc nodes now include leading whitespace in their location.
+6. The parser always parses a JSDocText node for comments in JSDoc. `string` is no longer part of the type of `comment`.
+7. In cases where Strada did produce a JSDocText node, Corsa no longer (incorrectly) includes all leading and trailing whitespace/asterisks, as well as initial `/**`.
+8. JSDocMemberName is now parsed as QualifiedName. These two nodes previously only differed by type, and now QualifiedName has a much less restrictive type for its left child.
 
 JSDoc types are parsed in normal type annotation position but show a grammar error. Corsa no longer parses the JSDoc types below, giving a parse error instead of a grammar error.
 

_packages/api/src/node.ts

@@ -20,6 +20,7 @@ declare module "@typescript/ast" {
 const popcount8 = [0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8];
 
 const childProperties: Readonly<Partial<Record<SyntaxKind, readonly string[]>>> = {
+    [SyntaxKind.SourceFile]: ["statements", "endOfFileToken"],
     [SyntaxKind.QualifiedName]: ["left", "right"],
     [SyntaxKind.TypeParameter]: ["modifiers", "name", "constraint", "defaultType"],
     [SyntaxKind.IfStatement]: ["expression", "thenStatement", "elseStatement"],
@@ -221,7 +222,7 @@ export class RemoteNodeBase {
 
     protected get childMask(): number {
         if (this.dataType !== NODE_DATA_TYPE_CHILDREN) {
-            return 0;
+            return -1;
         }
         return this.data & NODE_CHILD_MASK;
     }
@@ -674,6 +675,9 @@ export class RemoteNode extends RemoteNodeBase implements Node {
     get elseStatement(): RemoteNode | undefined {
         return this.getNamedChild("elseStatement") as RemoteNode;
     }
+    get endOfFileToken(): RemoteNode | undefined {
+        return this.getNamedChild("endOfFileToken") as RemoteNode;
+    }
     get equalsGreaterThanToken(): RemoteNode | undefined {
         return this.getNamedChild("equalsGreaterThanToken") as RemoteNode;
     }

_packages/api/test/api.test.ts

@@ -109,7 +109,7 @@ describe("SourceFile", () => {
             nodeCount++;
             node.forEachChild(visit);
         });
-        assert.equal(nodeCount, 7);
+        assert.equal(nodeCount, 8);
     });
 });
 

_packages/ast/src/nodes.ts

@@ -14,6 +14,7 @@ export interface Node extends ReadonlyTextRange {
 export interface SourceFile extends Node {
     readonly kind: SyntaxKind.SourceFile;
     readonly statements: NodeArray<Statement>;
+    readonly endOfFileToken: EndOfFile;
     readonly text: string;
     readonly fileName: string;
 }

internal/ast/ast.go

@@ -10001,10 +10001,11 @@ type SourceFile struct {
 	compositeNodeBase
 
 	// Fields set by NewSourceFile
-	fileName     string // For debugging convenience
-	parseOptions SourceFileParseOptions
-	text         string
-	Statements   *NodeList // NodeList[*Statement]
+	fileName       string // For debugging convenience
+	parseOptions   SourceFileParseOptions
+	text           string
+	Statements     *NodeList  // NodeList[*Statement]
+	EndOfFileToken *TokenNode // TokenNode[*EndOfFileToken]
 
 	// Fields set by parser
 	diagnostics                 []*Diagnostic
@@ -10053,7 +10054,7 @@ type SourceFile struct {
 	tokenCache   map[core.TextRange]*Node
 }
 
-func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, statements *NodeList) *Node {
+func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, statements *NodeList, endOfFileToken *TokenNode) *Node {
 	if (tspath.GetEncodedRootLength(opts.FileName) == 0 && !strings.HasPrefix(opts.FileName, "^/")) || opts.FileName != tspath.NormalizePath(opts.FileName) {
 		panic(fmt.Sprintf("fileName should be normalized and absolute: %q", opts.FileName))
 	}
@@ -10062,6 +10063,7 @@ func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, st
 	data.parseOptions = opts
 	data.text = text
 	data.Statements = statements
+	data.EndOfFileToken = endOfFileToken
 	return f.newNode(KindSourceFile, data)
 }
 
@@ -10122,11 +10124,11 @@ func (node *SourceFile) SetBindDiagnostics(diags []*Diagnostic) {
 }
 
 func (node *SourceFile) ForEachChild(v Visitor) bool {
-	return visitNodeList(v, node.Statements)
+	return visitNodeList(v, node.Statements) || visit(v, node.EndOfFileToken)
 }
 
 func (node *SourceFile) VisitEachChild(v *NodeVisitor) *Node {
-	return v.Factory.UpdateSourceFile(node, v.visitTopLevelStatements(node.Statements))
+	return v.Factory.UpdateSourceFile(node, v.visitTopLevelStatements(node.Statements), v.visitToken(node.EndOfFileToken))
 }
 
 func (node *SourceFile) IsJS() bool {
@@ -10155,7 +10157,7 @@ func (node *SourceFile) copyFrom(other *SourceFile) {
 }
 
 func (node *SourceFile) Clone(f NodeFactoryCoercible) *Node {
-	updated := f.AsNodeFactory().NewSourceFile(node.parseOptions, node.text, node.Statements)
+	updated := f.AsNodeFactory().NewSourceFile(node.parseOptions, node.text, node.Statements, node.EndOfFileToken)
 	newFile := updated.AsSourceFile()
 	newFile.copyFrom(node)
 	return cloneNode(updated, node.AsNode(), f.AsNodeFactory().hooks)
@@ -10165,9 +10167,9 @@ func (node *SourceFile) computeSubtreeFacts() SubtreeFacts {
 	return propagateNodeListSubtreeFacts(node.Statements, propagateSubtreeFacts)
 }
 
-func (f *NodeFactory) UpdateSourceFile(node *SourceFile, statements *StatementList) *Node {
-	if statements != node.Statements {
-		updated := f.NewSourceFile(node.parseOptions, node.text, statements).AsSourceFile()
+func (f *NodeFactory) UpdateSourceFile(node *SourceFile, statements *StatementList, endOfFileToken *TokenNode) *Node {
+	if statements != node.Statements || endOfFileToken != node.EndOfFileToken {
+		updated := f.NewSourceFile(node.parseOptions, node.text, statements, endOfFileToken).AsSourceFile()
 		updated.copyFrom(node)
 		return updateNode(updated.AsNode(), node.AsNode(), f.hooks)
 	}

internal/astnav/tokens.go

@@ -49,6 +49,9 @@ func getTokenAtPosition(
 	left := 0
 
 	testNode := func(node *ast.Node) int {
+		if node.Kind == ast.KindEndOfFile {
+			return 0
+		}
 		if node.End() == position && includePrecedingTokenAtEndPosition != nil {
 			prevSubtree = node
 		}
@@ -247,7 +250,7 @@ func FindPrecedingToken(sourceFile *ast.SourceFile, position int) *ast.Node {
 func FindPrecedingTokenEx(sourceFile *ast.SourceFile, position int, startNode *ast.Node, excludeJSDoc bool) *ast.Node {
 	var find func(node *ast.Node) *ast.Node
 	find = func(n *ast.Node) *ast.Node {
-		if ast.IsNonWhitespaceToken(n) {
+		if ast.IsNonWhitespaceToken(n) && n.Kind != ast.KindEndOfFile {
 			return n
 		}
 

internal/astnav/tokens_test.go

@@ -20,8 +20,6 @@ import (
 )
 
 var testFiles = []string{
-	// !!! EOFToken JSDoc parsing is missing
-	// filepath.Join(repo.TestDataPath, "fixtures/astnav/eofJSDoc.ts"),
 	filepath.Join(repo.TypeScriptSubmodulePath, "src/services/mapCode.ts"),
 }
 

internal/binder/binder.go

@@ -1637,8 +1637,9 @@ func (b *Binder) bindChildren(node *ast.Node) {
 	// case *JSDocImportTag:
 	// 	b.bindJSDocImportTag(node)
 	case ast.KindSourceFile:
-		b.bindEachStatementFunctionsFirst(node.AsSourceFile().Statements)
-		// b.bind(node.endOfFileToken)
+		sourceFile := node.AsSourceFile()
+		b.bindEachStatementFunctionsFirst(sourceFile.Statements)
+		b.bind(sourceFile.EndOfFileToken)
 	case ast.KindBlock:
 		b.bindEachStatementFunctionsFirst(node.AsBlock().Statements)
 	case ast.KindModuleBlock:

internal/fourslash/_scripts/failingTests.txt

@@ -1,4 +1,3 @@
-TestAsOperatorCompletion
 TestAutoImportsWithRootDirsAndRootedPath01
 TestClosedCommentsInConstructor
 TestCompletionCloneQuestionToken

internal/fourslash/tests/gen/asOperatorCompletion_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestAsOperatorCompletion(t *testing.T) {
 	t.Parallel()
-	t.Skip()
+
 	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
 	const content = `type T = number;
 var x;

internal/ls/completions.go

@@ -3659,7 +3659,6 @@ func tryGetObjectTypeDeclarationCompletionContainer(
 			return location.Parent
 		}
 		return nil
-	// !!! we don't include EOF token anymore, verify what we should do in this case.
 	case ast.KindEndOfFile:
 		stmtList := location.Parent.AsSourceFile().Statements
 		if stmtList != nil && len(stmtList.Nodes) > 0 && ast.IsObjectTypeDeclaration(stmtList.Nodes[len(stmtList.Nodes)-1]) {

internal/parser/parser.go

@@ -111,10 +111,11 @@ func ParseSourceFile(opts ast.SourceFileParseOptions, sourceText string, scriptK
 func (p *Parser) parseJSONText() *ast.SourceFile {
 	pos := p.nodePos()
 	var statements *ast.NodeList
+	var eof *ast.TokenNode
 
 	if p.token == ast.KindEndOfFile {
 		statements = p.newNodeList(core.NewTextRange(pos, p.nodePos()), nil)
-		p.parseTokenNode()
+		eof = p.parseTokenNode()
 	} else {
 		var expressions any // []*ast.Expression | *ast.Expression
 
@@ -167,9 +168,9 @@ func (p *Parser) parseJSONText() *ast.SourceFile {
 		statement := p.factory.NewExpressionStatement(expression)
 		p.finishNode(statement, pos)
 		statements = p.newNodeList(core.NewTextRange(pos, p.nodePos()), []*ast.Node{statement})
-		p.parseExpectedToken(ast.KindEndOfFile)
+		eof = p.parseExpectedToken(ast.KindEndOfFile)
 	}
-	node := p.factory.NewSourceFile(p.opts, p.sourceText, statements)
+	node := p.factory.NewSourceFile(p.opts, p.sourceText, statements, eof)
 	p.finishNode(node, pos)
 	result := node.AsSourceFile()
 	p.finishSourceFile(result, false)
@@ -317,11 +318,19 @@ func (p *Parser) parseSourceFileWorker() *ast.SourceFile {
 	}
 	pos := p.nodePos()
 	statements := p.parseListIndex(PCSourceElements, (*Parser).parseToplevelStatement)
+	end := p.nodePos()
+	endHasJSDoc := p.hasPrecedingJSDocComment()
 	eof := p.parseTokenNode()
+	p.withJSDoc(eof, endHasJSDoc)
 	if eof.Kind != ast.KindEndOfFile {
 		panic("Expected end of file token from scanner.")
 	}
-	node := p.factory.NewSourceFile(p.opts, p.sourceText, statements)
+	if len(p.reparseList) > 0 {
+		statements = append(statements, p.reparseList...)
+		p.reparseList = nil
+		end = p.nodePos()
+	}
+	node := p.factory.NewSourceFile(p.opts, p.sourceText, p.newNodeList(core.NewTextRange(pos, end), statements), eof)
 	p.finishNode(node, pos)
 	result := node.AsSourceFile()
 	p.finishSourceFile(result, isDeclarationFile)
@@ -467,11 +476,10 @@ func (p *Parser) reparseTopLevelAwait(sourceFile *ast.SourceFile) *ast.Node {
 		}
 	}
 
-	return p.factory.NewSourceFile(sourceFile.ParseOptions(), p.sourceText, p.newNodeList(sourceFile.Statements.Loc, statements))
+	return p.factory.NewSourceFile(sourceFile.ParseOptions(), p.sourceText, p.newNodeList(sourceFile.Statements.Loc, statements), sourceFile.EndOfFileToken)
 }
 
-func (p *Parser) parseListIndex(kind ParsingContext, parseElement func(p *Parser, index int) *ast.Node) *ast.NodeList {
-	pos := p.nodePos()
+func (p *Parser) parseListIndex(kind ParsingContext, parseElement func(p *Parser, index int) *ast.Node) []*ast.Node {
 	saveParsingContexts := p.parsingContexts
 	p.parsingContexts |= 1 << kind
 	list := make([]*ast.Node, 0, 16)
@@ -492,11 +500,13 @@ func (p *Parser) parseListIndex(kind ParsingContext, parseElement func(p *Parser
 	p.parsingContexts = saveParsingContexts
 	slice := p.nodeSlicePool.NewSlice(len(list))
 	copy(slice, list)
-	return p.newNodeList(core.NewTextRange(pos, p.nodePos()), slice)
+	return slice
 }
 
 func (p *Parser) parseList(kind ParsingContext, parseElement func(p *Parser) *ast.Node) *ast.NodeList {
-	return p.parseListIndex(kind, func(p *Parser, _ int) *ast.Node { return parseElement(p) })
+	pos := p.nodePos()
+	nodes := p.parseListIndex(kind, func(p *Parser, _ int) *ast.Node { return parseElement(p) })
+	return p.newNodeList(core.NewTextRange(pos, p.nodePos()), nodes)
 }
 
 // Return a non-nil (but possibly empty) slice if parsing was successful, or nil if parseElement returned nil

internal/parser/reparser.go

@@ -102,6 +102,9 @@ func (p *Parser) reparseUnhosted(tag *ast.Node, parent *ast.Node, jsDoc *ast.Nod
 	case ast.KindJSDocImportTag:
 		importTag := tag.AsJSDocImportTag()
 		importClause := importTag.ImportClause
+		if importClause == nil {
+			break
+		}
 		importClause.Flags |= ast.NodeFlagsReparsed
 		importClause.AsImportClause().IsTypeOnly = true
 		importDeclaration := p.factory.NewJSImportDeclaration(importTag.Modifiers(), importClause, importTag.ModuleSpecifier, importTag.Attributes)