Add a conditional compilation flag to verify token choices

Verification of token choice is currently causing assertion failures where the list of expected token choices in the syntax tree doesn't match those the parser generates.

I’d still like to get the verification code into `main` so I can conditionally enable it locally while working on the issues, so add it behind a conditional compilation flag.

Author: ahoppen

Sources/SwiftSyntax/Raw/RawSyntaxValidation.swift.gyb

@@ -25,16 +25,33 @@
 /// Results in an assertion failure if the layout is invalid.
 func validateLayout(layout: RawSyntaxBuffer, as kind: SyntaxKind) {
 #if DEBUG
+  enum TokenChoice: CustomStringConvertible {
+    case kind(RawTokenBaseKind)
+    case keyword(StaticString)
+
+    var description: String {
+      switch self {
+      case .kind(let kind):
+        return "\(kind)"
+      case .keyword(let keyword):
+        return "keyword('\(keyword)')"
+      }
+    }
+  }
+
   enum ValidationError: CustomStringConvertible {
     case expectedNonNil(expectedKind: RawSyntaxNodeProtocol.Type, file: StaticString, line: UInt)
     case kindMismatch(expectedKind: RawSyntaxNodeProtocol.Type, actualKind: SyntaxKind, file: StaticString, line: UInt)
+    case tokenMismatch(expectedTokenChoices: [TokenChoice], actualKind: RawTokenBaseKind, actualText: SyntaxText, file: StaticString, line: UInt)
 
     var description: String {
       switch self {
       case .expectedNonNil(expectedKind: let expectedKind, file: _, line: _):
         return "Expected non-nil node of type \(expectedKind) but received nil"
       case .kindMismatch(expectedKind: let expectedKind, actualKind: let actualKind, file: _, line: _):
         return "Expected node of type \(expectedKind) but received \(actualKind)"
+      case .tokenMismatch(expectedTokenChoices: let tokenChoices, actualKind: let actualKind, actualText: let actualText, file: _, line: _):
+        return "Expected token with one of \(tokenChoices) but received \(actualKind) with text '\(actualText)'"
       }
     }
 
@@ -44,6 +61,8 @@ func validateLayout(layout: RawSyntaxBuffer, as kind: SyntaxKind) {
         return (file, line)
       case .kindMismatch(expectedKind: _, actualKind: _, file: let file, line: let line):
         return (file, line)
+      case .tokenMismatch(expectedTokenChoices: _, actualKind: _, actualText: _, file: let file, line: let line):
+        return (file, line)
       }
     }
   }
@@ -65,6 +84,52 @@ func validateLayout(layout: RawSyntaxBuffer, as kind: SyntaxKind) {
     return nil
   }
 
+  func verify(_ raw: RawSyntax?, as _: RawTokenSyntax?.Type, tokenChoices: [TokenChoice], file: StaticString = #file, line: UInt = #line) -> ValidationError? {
+    // Validation of token choice is currently causing assertion failures where
+    // the list of expected token choices in the syntax tree doesn't match those
+    // the parser generates. Disable the verification for now until all issues
+    // regarding it are fixed.
+    #if VALIDATE_TOKEN_CHOICES
+    if raw != nil {
+      return verify(raw, as: RawTokenSyntax.self, tokenChoices: tokenChoices, file: file, line: line)
+    }
+    return nil
+    #else
+    return verify(raw, as RawTokenSyntax?.self)
+    #endif
+  }
+
+  func verify(_ raw: RawSyntax?, as _: RawTokenSyntax.Type, tokenChoices: [TokenChoice], file: StaticString = #file, line: UInt = #line) -> ValidationError? {
+    // Validation of token choice is currently causing assertion failures where
+    // the list of expected token choices in the syntax tree doesn't match those
+    // the parser generates. Disable the verification for now until all issues
+    // regarding it are fixed.
+    #if VALIDATE_TOKEN_CHOICES
+    guard let raw = raw else {
+      return .expectedNonNil(expectedKind: RawTokenSyntax.self, file: file, line: line)
+    }
+    if let error = verify(raw, as: RawTokenSyntax?.self) {
+      return error
+    }
+    let tokenView = raw.tokenView!
+    for tokenChoice in tokenChoices {
+      switch tokenChoice {
+      case .kind(let kind):
+        if raw.tokenView?.rawKind.base == kind {
+          return nil
+        }
+      case .keyword(let keyword):
+        if tokenView.rawKind.base == .keyword && tokenView.rawText == SyntaxText(keyword) {
+          return nil
+        }
+      }
+    }
+    return ValidationError.tokenMismatch(expectedTokenChoices: tokenChoices, actualKind: tokenView.rawKind.base, actualText: tokenView.rawText, file: file, line: line)
+    #else
+    return verify(raw, as RawTokenSyntax.self)
+    #endif
+  }
+
   func assertNoError(_ nodeKind: SyntaxKind, _ index: Int, _ error: ValidationError?) {
     if let error = error {
       let (file, line) = error.fileAndLine
@@ -101,10 +166,33 @@ func validateLayout(layout: RawSyntaxBuffer, as kind: SyntaxKind) {
 %       if child.node_choices:
     assertAnyHasNoError(kind, ${idx}, [
 %         for node_choice in child.node_choices:
-      verify(layout[${idx}], as: Raw${child.type_name}${"?" if child.is_optional else ""}.self),
+%           if node_choice.token_choices:
+%             token_choices = []
+%             for (token, text) in node_choice.token_choices:
+%               if token.name == 'Keyword' and text:
+%                 token_choices.append(f'.keyword("{text}")')
+%               else:
+%                 token_choices.append((f'.kind(.{token.swift_kind()})'))
+%               end
+%             end
+      verify(layout[${idx}], as: Raw${node_choice.type_name}${"?" if child.is_optional else ""}.self, tokenChoices: [${', '.join(token_choices)}]),
+%           else:
+      verify(layout[${idx}], as: Raw${node_choice.type_name}${"?" if child.is_optional else ""}.self),
+%           end
 %         end
     ])
 %       else:
+%         if child.token_choices:
+%           token_choices = []
+%           for (token, text) in child.token_choices:
+%             if token.name == 'Keyword' and text:
+%               token_choices.append(f'.keyword("{text}")')
+%             else:
+%               token_choices.append((f'.kind(.{token.swift_kind()})'))
+%             end
+%           end
+    assertNoError(kind, ${idx}, verify(layout[${idx}], as: Raw${child.type_name}${"?" if child.is_optional else ""}.self, tokenChoices: [${', '.join(token_choices)}]))
+%         end
     assertNoError(kind, ${idx}, verify(layout[${idx}], as: Raw${child.type_name}${"?" if child.is_optional else ""}.self))
 %       end
 %     end