Merge pull request #1320 from ahoppen/ahoppen/rename-lexererror

Rename `LexerError` -> `TokenDiagnostic`

Author: ahoppen

Sources/SwiftParser/Lexer/Cursor.swift

@@ -40,7 +40,7 @@ extension Lexer {
     @_spi(RawSyntax)
     public var rawTokenKind: RawTokenKind
     public var flags: Lexeme.Flags
-    public var error: LexerError?
+    public var diagnostic: TokenDiagnostic?
     var start: UnsafePointer<UInt8>
     public var leadingTriviaByteLength: Int
     public var textByteLength: Int
@@ -60,7 +60,7 @@ extension Lexer {
     init(
       tokenKind: RawTokenKind,
       flags: Flags,
-      error: LexerError?,
+      diagnostic: TokenDiagnostic?,
       start: UnsafePointer<UInt8>,
       leadingTriviaLength: Int,
       textLength: Int,
@@ -69,7 +69,7 @@ extension Lexer {
     ) {
       self.rawTokenKind = tokenKind
       self.flags = flags
-      self.error = error
+      self.diagnostic = diagnostic
       self.start = start
       self.leadingTriviaByteLength = leadingTriviaLength
       self.textByteLength = textLength

Sources/SwiftParser/Lexer/Lexeme.swift

@@ -47,7 +47,7 @@ extension Lexer {
           self.nextToken = Lexeme(
             tokenKind: .eof,
             flags: [],
-            error: nil,
+            diagnostic: nil,
             start: self.cursor.pointer,
             leadingTriviaLength: 0,
             textLength: 0,

Sources/SwiftParser/Lexer/LexemeSequence.swift

@@ -187,7 +187,7 @@ public struct Parser {
       wholeText: tok.wholeText,
       textRange: tok.textRange,
       presence: .present,
-      lexerError: tok.error,
+      tokenDiagnostic: tok.diagnostic,
       arena: arena
     )
   }
@@ -511,18 +511,18 @@ extension Parser {
     assert(tokenText.hasPrefix(prefix))
 
     let endIndex = current.textRange.lowerBound.advanced(by: prefix.count)
-    var lexerError = current.error
-    if let error = lexerError, error.byteOffset > prefix.count + current.leadingTriviaByteLength {
+    var tokenDiagnostic = current.diagnostic
+    if let error = tokenDiagnostic, error.byteOffset > prefix.count + current.leadingTriviaByteLength {
       // The lexer error isn't in the prefix. Drop it.
-      lexerError = nil
+      tokenDiagnostic = nil
     }
 
     let tok = RawTokenSyntax(
       kind: tokenKind,
       wholeText: SyntaxText(rebasing: current.wholeText[..<endIndex]),
       textRange: current.textRange.lowerBound..<endIndex,
       presence: .present,
-      lexerError: lexerError,
+      tokenDiagnostic: tokenDiagnostic,
       arena: self.arena
     )
 

Sources/SwiftParser/Parser.swift

@@ -55,13 +55,13 @@ fileprivate class StringLiteralExpressionIndentationChecker {
     if hasSufficientIndentation {
       return nil
     }
-    if token.tokenView.lexerError != nil {
+    if token.tokenView.tokenDiagnostic != nil {
       // Token already has a lexer error, ignore the indentation error until that
       // error is fixed
       return nil
     }
-    return token.tokenView.withLexerError(
-      lexerError: LexerError(.insufficientIndentationInMultilineStringLiteral, byteOffset: 0),
+    return token.tokenView.withTokenDiagnostic(
+      tokenDiagnostic: TokenDiagnostic(.insufficientIndentationInMultilineStringLiteral, byteOffset: 0),
       arena: arena
     )
   }
@@ -136,7 +136,7 @@ extension Parser {
     in token: RawTokenSyntax,
     leading reclassifyLeading: SyntaxText = "",
     trailing reclassifyTrailing: SyntaxText = "",
-    lexerError: LexerError? = nil
+    tokenDiagnostic: TokenDiagnostic? = nil
   ) -> RawTokenSyntax {
     assert(SyntaxText(rebasing: token.tokenText.prefix(reclassifyLeading.count)) == reclassifyLeading)
     assert(SyntaxText(rebasing: token.tokenText.suffix(reclassifyTrailing.count)) == reclassifyTrailing)
@@ -146,7 +146,7 @@ extension Parser {
       leadingTriviaPieces: token.leadingTriviaPieces + TriviaParser.parseTrivia(reclassifyLeading, position: .trailing),
       trailingTriviaPieces: TriviaParser.parseTrivia(reclassifyTrailing, position: .trailing) + token.trailingTriviaPieces,
       presence: token.presence,
-      lexerError: token.tokenView.lexerError ?? lexerError,
+      tokenDiagnostic: token.tokenView.tokenDiagnostic ?? tokenDiagnostic,
       arena: self.arena
     )
   }
@@ -253,8 +253,8 @@ extension Parser {
             // Empty lines don't need to be indented and there's no indentation we need to strip away.
           } else {
             let actualIndentation = SyntaxText(rebasing: segment.content.tokenText.prefix(while: { $0 == UInt8(ascii: " ") || $0 == UInt8(ascii: "\t") }))
-            let lexerError = LexerError(.insufficientIndentationInMultilineStringLiteral, byteOffset: 0)
-            let content = self.reclassifyTrivia(in: segment.content, leading: actualIndentation, lexerError: lexerError)
+            let tokenDiagnostic = TokenDiagnostic(.insufficientIndentationInMultilineStringLiteral, byteOffset: 0)
+            let content = self.reclassifyTrivia(in: segment.content, leading: actualIndentation, tokenDiagnostic: tokenDiagnostic)
             segment = RawStringSegmentSyntax(
               segment.unexpectedBeforeContent,
               content: content,
@@ -372,7 +372,7 @@ extension Parser {
         leadingTriviaPieces: parsedTrivia + closeQuote.leadingTriviaPieces,
         trailingTriviaPieces: closeQuote.trailingTriviaPieces,
         presence: closeQuote.presence,
-        lexerError: closeQuote.tokenView.lexerError,
+        tokenDiagnostic: closeQuote.tokenView.tokenDiagnostic,
         arena: self.arena
       )
     } else {

Sources/SwiftParser/StringLiterals.swift

@@ -16,11 +16,11 @@ import SwiftDiagnostics
 fileprivate let diagnosticDomain: String = "SwiftLexer"
 
 /// A error diagnostic whose ID is determined by the diagnostic's type.
-public protocol LexerError: DiagnosticMessage {
+public protocol TokenError: DiagnosticMessage {
   var diagnosticID: MessageID { get }
 }
 
-public extension LexerError {
+public extension TokenError {
   static var diagnosticID: MessageID {
     return MessageID(domain: diagnosticDomain, id: "\(self)")
   }
@@ -37,7 +37,7 @@ public extension LexerError {
 // MARK: - Errors (please sort alphabetically)
 
 /// Please order the cases in this enum alphabetically by case name.
-public enum StaticLexerError: String, DiagnosticMessage {
+public enum StaticTokenError: String, DiagnosticMessage {
   case expectedBinaryExponentInHexFloatLiteral = "hexadecimal floating point literal must end with an exponent"
   case expectedClosingBraceInUnicodeEscape = #"expected '}' in \u{...} escape sequence"#
   case expectedDigitInFloatLiteral = "expected a digit in floating point exponent"
@@ -48,7 +48,7 @@ public enum StaticLexerError: String, DiagnosticMessage {
   case invalidIdentifierStartCharacter = "an identifier cannot begin with this character"
   case invalidNumberOfHexDigitsInUnicodeEscape = #"\u{...} escape sequence expects between 1 and 8 hex digits"#
   case invalidUtf8 = "invalid UTF-8 found in source file"
-  case lexerErrorOffsetOverflow = "the lexer dicovered an error in this token but was not able to represent its offset due to overflow; please split the token"
+  case tokenDiagnosticOffsetOverflow = "the lexer dicovered an error in this token but was not able to represent its offset due to overflow; please split the token"
   case sourceConflictMarker = "source control conflict marker in source file"
   case unexpectedBlockCommentEnd = "unexpected end of block comment"
   case unicodeCurlyQuote = #"unicode curly quote found; use '"' instead"#
@@ -64,7 +64,7 @@ public enum StaticLexerError: String, DiagnosticMessage {
 }
 
 /// Please order the cases in this enum alphabetically by case name.
-public enum StaticLexerWarning: String, DiagnosticMessage {
+public enum StaticTokenWarning: String, DiagnosticMessage {
   case nonBreakingSpace = "non-breaking space (U+00A0) used instead of regular space"
   case nulCharacter = "nul character embedded in middle of file"
 
@@ -77,7 +77,7 @@ public enum StaticLexerWarning: String, DiagnosticMessage {
   public var severity: DiagnosticSeverity { .warning }
 }
 
-public struct InvalidFloatingPointExponentDigit: LexerError {
+public struct InvalidFloatingPointExponentDigit: TokenError {
   public enum Kind {
     case digit(Unicode.Scalar)
     case character(Unicode.Scalar)
@@ -94,7 +94,7 @@ public struct InvalidFloatingPointExponentDigit: LexerError {
   }
 }
 
-public struct InvalidDigitInIntegerLiteral: LexerError {
+public struct InvalidDigitInIntegerLiteral: TokenError {
   public enum Kind {
     case binary(Unicode.Scalar)
     case octal(Unicode.Scalar)
@@ -118,10 +118,10 @@ public struct InvalidDigitInIntegerLiteral: LexerError {
   }
 }
 
-// MARK: - Convert LexerError from SwiftSyntax to error messages
+// MARK: - Convert TokenDiagnostic from SwiftSyntax to error messages
 
-public extension SwiftSyntax.LexerError {
-  /// `tokenText` is the entire text of the token in which the `LexerError`
+public extension SwiftSyntax.TokenDiagnostic {
+  /// `tokenText` is the entire text of the token in which the `TokenDiagnostic`
   /// occurred, including trivia.
   @_spi(RawSyntax)
   func diagnosticMessage(wholeTextBytes: [UInt8]) -> DiagnosticMessage {
@@ -132,34 +132,34 @@ public extension SwiftSyntax.LexerError {
     }
 
     switch self.kind {
-    case .expectedBinaryExponentInHexFloatLiteral: return StaticLexerError.expectedBinaryExponentInHexFloatLiteral
-    case .expectedClosingBraceInUnicodeEscape: return StaticLexerError.expectedClosingBraceInUnicodeEscape
-    case .expectedDigitInFloatLiteral: return StaticLexerError.expectedDigitInFloatLiteral
-    case .expectedHexCodeInUnicodeEscape: return StaticLexerError.expectedHexCodeInUnicodeEscape
-    case .expectedHexDigitInHexLiteral: return StaticLexerError.expectedHexDigitInHexLiteral
+    case .expectedBinaryExponentInHexFloatLiteral: return StaticTokenError.expectedBinaryExponentInHexFloatLiteral
+    case .expectedClosingBraceInUnicodeEscape: return StaticTokenError.expectedClosingBraceInUnicodeEscape
+    case .expectedDigitInFloatLiteral: return StaticTokenError.expectedDigitInFloatLiteral
+    case .expectedHexCodeInUnicodeEscape: return StaticTokenError.expectedHexCodeInUnicodeEscape
+    case .expectedHexDigitInHexLiteral: return StaticTokenError.expectedHexDigitInHexLiteral
     case .insufficientIndentationInMultilineStringLiteral:
       // This should be diagnosed when visiting the `StringLiteralExprSyntax`
       // inside `ParseDiagnosticsGenerator` but fall back to an error message
       // here in case the error is not diagnosed.
       return InvalidIndentationInMultiLineStringLiteralError(kind: .insufficientIndentation, lines: 1)
     case .invalidBinaryDigitInIntegerLiteral: return InvalidDigitInIntegerLiteral(kind: .binary(scalarAtErrorOffset))
-    case .invalidCharacter: return StaticLexerError.invalidCharacter
+    case .invalidCharacter: return StaticTokenError.invalidCharacter
     case .invalidDecimalDigitInIntegerLiteral: return InvalidDigitInIntegerLiteral(kind: .decimal(scalarAtErrorOffset))
-    case .invalidEscapeSequenceInStringLiteral: return StaticLexerError.invalidEscapeSequenceInStringLiteral
+    case .invalidEscapeSequenceInStringLiteral: return StaticTokenError.invalidEscapeSequenceInStringLiteral
     case .invalidFloatingPointExponentCharacter: return InvalidFloatingPointExponentDigit(kind: .character(scalarAtErrorOffset))
     case .invalidFloatingPointExponentDigit: return InvalidFloatingPointExponentDigit(kind: .digit(scalarAtErrorOffset))
     case .invalidHexDigitInIntegerLiteral: return InvalidDigitInIntegerLiteral(kind: .hex(scalarAtErrorOffset))
-    case .invalidIdentifierStartCharacter: return StaticLexerError.invalidIdentifierStartCharacter
-    case .invalidNumberOfHexDigitsInUnicodeEscape: return StaticLexerError.invalidNumberOfHexDigitsInUnicodeEscape
+    case .invalidIdentifierStartCharacter: return StaticTokenError.invalidIdentifierStartCharacter
+    case .invalidNumberOfHexDigitsInUnicodeEscape: return StaticTokenError.invalidNumberOfHexDigitsInUnicodeEscape
     case .invalidOctalDigitInIntegerLiteral: return InvalidDigitInIntegerLiteral(kind: .octal(scalarAtErrorOffset))
-    case .invalidUtf8: return StaticLexerError.invalidUtf8
-    case .lexerErrorOffsetOverflow: return StaticLexerError.lexerErrorOffsetOverflow
-    case .nonBreakingSpace: return StaticLexerWarning.nonBreakingSpace
-    case .nulCharacter: return StaticLexerWarning.nulCharacter
-    case .sourceConflictMarker: return StaticLexerError.sourceConflictMarker
-    case .unexpectedBlockCommentEnd: return StaticLexerError.unexpectedBlockCommentEnd
-    case .unicodeCurlyQuote: return StaticLexerError.unicodeCurlyQuote
-    case .unprintableAsciiCharacter: return StaticLexerError.unprintableAsciiCharacter
+    case .invalidUtf8: return StaticTokenError.invalidUtf8
+    case .tokenDiagnosticOffsetOverflow: return StaticTokenError.tokenDiagnosticOffsetOverflow
+    case .nonBreakingSpace: return StaticTokenWarning.nonBreakingSpace
+    case .nulCharacter: return StaticTokenWarning.nulCharacter
+    case .sourceConflictMarker: return StaticTokenError.sourceConflictMarker
+    case .unexpectedBlockCommentEnd: return StaticTokenError.unexpectedBlockCommentEnd
+    case .unicodeCurlyQuote: return StaticTokenError.unicodeCurlyQuote
+    case .unprintableAsciiCharacter: return StaticTokenError.unprintableAsciiCharacter
     }
   }
 

Sources/SwiftParserDiagnostics/LexerDiagnosticMessages.swift

@@ -144,7 +144,7 @@ final class MultiLineStringLiteralIndentatinDiagnosticsGenerator: SyntaxVisitor
       return .visitChildren
     }
 
-    if token.lexerError?.kind == .insufficientIndentationInMultilineStringLiteral {
+    if token.tokenDiagnostic?.kind == .insufficientIndentationInMultilineStringLiteral {
       addIncorrectlyIndentedToken(token: token)
     } else {
       finishInProgressDiagnostic()

Sources/SwiftParserDiagnostics/MultiLineStringLiteralDiagnoticsGenerator.swift

@@ -31,7 +31,7 @@ fileprivate extension TokenSyntax {
 }
 
 fileprivate extension DiagnosticSeverity {
-  func matches(_ lexerErorSeverity: SwiftSyntax.LexerError.Severity) -> Bool {
+  func matches(_ lexerErorSeverity: SwiftSyntax.TokenDiagnostic.Severity) -> Bool {
     switch (self, lexerErorSeverity) {
     case (.error, .error):
       return true
@@ -359,14 +359,14 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     if token.presence == .missing {
       handleMissingToken(token)
     } else {
-      if let lexerError = token.lexerError {
-        let message = lexerError.diagnosticMessage(in: token)
-        assert(message.severity.matches(lexerError.severity))
+      if let tokenDiagnostic = token.tokenDiagnostic {
+        let message = tokenDiagnostic.diagnosticMessage(in: token)
+        assert(message.severity.matches(tokenDiagnostic.severity))
         self.addDiagnostic(
           token,
-          position: token.position.advanced(by: Int(lexerError.byteOffset)),
+          position: token.position.advanced(by: Int(tokenDiagnostic.byteOffset)),
           message,
-          fixIts: lexerError.fixIts(in: token)
+          fixIts: tokenDiagnostic.fixIts(in: token)
         )
       }
     }

Sources/SwiftParserDiagnostics/ParseDiagnosticsGenerator.swift

@@ -11,7 +11,6 @@ add_swift_host_library(SwiftSyntax
   BumpPtrAllocator.swift
   CommonAncestor.swift
   IncrementalParseTransition.swift
-  LexerError.swift
   SourceLength.swift
   SourceLocation.swift
   SourcePresence.swift
@@ -22,6 +21,7 @@ add_swift_host_library(SwiftSyntax
   SyntaxOtherNodes.swift
   SyntaxText.swift
   SyntaxTreeViewMode.swift
+  TokenDiagnostic.swift
   Utils.swift
 
   Raw/RawSyntax.swift