Rename eof to endOfFile (#1842)

Update Codegeneration to use 'endOfFile' instead of 'eof', and include EOF tokens as part of the normal SYNTAX_TOKENS

Author: mininny

CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift

@@ -2036,7 +2036,7 @@ public let DECL_NODES: [Node] = [
     ]
   ),
 
-  // source-file = code-block-item-list eof
+  // source-file = code-block-item-list endOfFile
   Node(
     kind: .sourceFile,
     base: .syntax,
@@ -2051,7 +2051,7 @@ public let DECL_NODES: [Node] = [
       Child(
         name: "EndOfFileToken",
         deprecatedName: "EOFToken",
-        kind: .token(choices: [.token(tokenKind: "EOFToken")])
+        kind: .token(choices: [.token(tokenKind: "EndOfFileToken")])
       ),
     ]
   ),

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift

@@ -183,6 +183,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PunctuatorSpec(name: "Comma", kind: "comma", text: ",", requiresTrailingSpace: true),
   MiscSpec(name: "DollarIdentifier", kind: "dollarident", nameForDiagnostics: "dollar identifier", classification: "DollarIdentifier"),
   PunctuatorSpec(name: "Ellipsis", kind: "ellipsis", text: "..."),
+  MiscSpec(name: "EndOfFile", kind: "eof", nameForDiagnostics: "end of file", text: ""),
   PunctuatorSpec(name: "Equal", kind: "equal", text: "=", requiresLeadingSpace: true, requiresTrailingSpace: true),
   PunctuatorSpec(name: "ExclamationMark", kind: "exclaim_postfix", text: "!"),
   MiscSpec(name: "ExtendedRegexDelimiter", kind: "extended_regex_delimiter", nameForDiagnostics: "extended delimiter", classification: "RegexLiteral"),
@@ -224,8 +225,6 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   MiscSpec(name: "Wildcard", kind: "_", nameForDiagnostics: "wildcard", text: "_"),
 ]
 
-// FIXME: Generate the EOF token as part of the normal SYNTAX_TOKENS and remove the special handling for it.
 public let SYNTAX_TOKEN_MAP = Dictionary(
-  uniqueKeysWithValues: (SYNTAX_TOKENS + [MiscSpec(name: "EOF", kind: "eof", nameForDiagnostics: "end of file", text: "")])
-    .map { ("\($0.name)Token", $0) }
+  uniqueKeysWithValues: SYNTAX_TOKENS.map { ("\($0.name)Token", $0) }
 )

CodeGeneration/Sources/Utils/SyntaxBuildableChild.swift

@@ -74,9 +74,6 @@ public extension Child {
     if token.isKeyword {
       return InitializerClauseSyntax(value: ExprSyntax(".\(raw: token.swiftKind)()"))
     }
-    if token.name == "EOF" {
-      return InitializerClauseSyntax(value: ExprSyntax(".eof()"))
-    }
     if token.text != nil {
       return InitializerClauseSyntax(value: ExprSyntax(".\(raw: token.swiftKind)Token()"))
     }

CodeGeneration/Sources/Utils/SyntaxBuildableType.swift

@@ -68,8 +68,6 @@ public struct SyntaxBuildableType: Hashable {
       } else if token.text != nil {
         return ExprSyntax(".\(raw: lowercaseFirstWord(name: token.name))Token()")
       }
-    } else if case .token("EOFToken") = kind {
-      return ExprSyntax(".eof()")
     }
     return nil
   }

CodeGeneration/Sources/generate-swiftsyntax/templates/ideutils/SyntaxClassificationFile.swift

@@ -77,13 +77,10 @@ let syntaxClassificationFile = SourceFileSyntax(leadingTrivia: copyrightHeader)
             if let classification = token.classification {
               StmtSyntax("return .\(raw: classification.swiftName)")
             } else {
-              StmtSyntax("return .none)")
+              StmtSyntax("return .none")
             }
           }
         }
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return .none")
-        }
       }
     }
   }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/IsLexerClassifiedFile.swift

@@ -58,10 +58,6 @@ let isLexerClassifiedFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try! SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return false")
-        }
-
         for token in SYNTAX_TOKENS where token.isKeyword {
           SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
             StmtSyntax("return true")

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/TokenSpecStaticMembersFile.swift

@@ -19,8 +19,6 @@ let tokenSpecStaticMembersFile = SourceFileSyntax(leadingTrivia: copyrightHeader
   DeclSyntax("@_spi(RawSyntax) import SwiftSyntax")
 
   try! ExtensionDeclSyntax("extension TokenSpec") {
-    DeclSyntax("static var eof: TokenSpec { return TokenSpec(.eof) }")
-
     for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
       DeclSyntax("static var \(raw: token.swiftKind): TokenSpec { return TokenSpec(.\(raw: token.swiftKind)) }")
     }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparserdiagnostics/TokenNameForDiagnosticsFile.swift

@@ -21,10 +21,6 @@ let tokenNameForDiagnosticFile = SourceFileSyntax(leadingTrivia: copyrightHeader
   try! ExtensionDeclSyntax("extension TokenKind") {
     try! VariableDeclSyntax("var nameForDiagnostics: String") {
       try! SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax(#"return "end of file""#)
-        }
-
         for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
           SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
             StmtSyntax("return #\"\(raw: token.nameForDiagnostics)\"#")

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/TokenKindFile.swift

@@ -22,8 +22,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     public enum TokenKind: Hashable
     """
   ) {
-    DeclSyntax("case eof")
-
     for token in SYNTAX_TOKENS {
       // Tokens that don't have a set text have an associated value that
       // contains their text.
@@ -59,10 +57,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
             }
           }
         }
-
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax(#"return """#)
-        }
       }
     }
 
@@ -86,9 +80,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
           }
         }
 
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax(#"return """#)
-        }
         SwitchCaseSyntax("default:") {
           StmtSyntax(#"return """#)
         }
@@ -106,10 +97,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return false")
-        }
-
         for token in SYNTAX_TOKENS {
           SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
             StmtSyntax("return \(raw: type(of: token) == PunctuatorSpec.self)")
@@ -122,10 +109,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ExtensionDeclSyntax("extension TokenKind: Equatable") {
     try FunctionDeclSyntax("public static func ==(lhs: TokenKind, rhs: TokenKind) -> Bool") {
       try SwitchExprSyntax("switch (lhs, rhs)") {
-        SwitchCaseSyntax("case (.eof, .eof):") {
-          StmtSyntax("return true")
-        }
-
         for token in SYNTAX_TOKENS {
           if token.text != nil {
             SwitchCaseSyntax("case (.\(raw: token.swiftKind), .\(raw: token.swiftKind)):") {
@@ -156,8 +139,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     public enum RawTokenKind: UInt8, Equatable, Hashable
     """
   ) {
-    DeclSyntax("case eof")
-
     for token in SYNTAX_TOKENS {
       DeclSyntax("case \(raw: token.swiftKind)")
     }
@@ -169,10 +150,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try! SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax(#"return """#)
-        }
-
         for token in SYNTAX_TOKENS {
           if let text = token.text {
             SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
@@ -198,10 +175,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try! SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return false")
-        }
-
         for token in SYNTAX_TOKENS {
           SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
             StmtSyntax("return \(raw: type(of: token) == PunctuatorSpec.self)")
@@ -220,10 +193,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try! SwitchExprSyntax("switch rawKind") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return .eof")
-        }
-
         for token in SYNTAX_TOKENS {
           if token.swiftKind == "keyword" {
             SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
@@ -259,10 +228,6 @@ let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       try! SwitchExprSyntax("switch self") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return (.eof, nil)")
-        }
-
         for token in SYNTAX_TOKENS {
           if token.swiftKind == "keyword" {
             SwitchCaseSyntax("case .\(raw: token.swiftKind)(let keyword):") {

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/TokensFile.swift

@@ -90,21 +90,5 @@ let tokensFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         )
       }
     }
-
-    DeclSyntax(
-      """
-      public static func eof(
-        leadingTrivia: Trivia = [],
-        presence: SourcePresence = .present
-      ) -> TokenSyntax {
-        return TokenSyntax(
-          .eof,
-          leadingTrivia: leadingTrivia,
-          trailingTrivia: [],
-          presence: presence
-        )
-      }
-      """
-    )
   }
 }