Introduce ExperimentalFeatures

These allow the parser to optionally support new
Swift features that haven't yet been enabled by
default.

Author: hamishknight

Sources/SwiftParser/CMakeLists.txt

@@ -13,6 +13,7 @@ add_swift_host_library(SwiftParser
   CollectionNodes+Parsable.swift
   Declarations.swift
   Directives.swift
+  ExperimentalFeatures.swift
   Expressions.swift
   IncrementalParseTransition.swift
   Lookahead.swift

Sources/SwiftParser/ExperimentalFeatures.swift

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+extension Parser {
+  public struct ExperimentalFeatures: OptionSet {
+    public let rawValue: UInt
+    public init(rawValue: UInt) {
+      self.rawValue = rawValue
+    }
+  }
+}

Sources/SwiftParser/Lookahead.swift

@@ -27,22 +27,36 @@ extension Parser {
     /// i.e. how far it looked ahead.
     var tokensConsumed: Int = 0
 
+    /// The experimental features that have been enabled in the underlying
+    /// parser.
+    let experimentalFeatures: ExperimentalFeatures
+
     private init(
       lexemes: Lexer.LexemeSequence,
-      currentToken: Lexer.Lexeme
+      currentToken: Lexer.Lexeme,
+      experimentalFeatures: ExperimentalFeatures
     ) {
       self.lexemes = lexemes
       self.currentToken = currentToken
+      self.experimentalFeatures = experimentalFeatures
     }
 
     fileprivate init(cloning other: Parser) {
-      self.init(lexemes: other.lexemes, currentToken: other.currentToken)
+      self.init(
+        lexemes: other.lexemes,
+        currentToken: other.currentToken,
+        experimentalFeatures: other.experimentalFeatures
+      )
     }
 
     /// Initiates a lookahead session from the current point in this
     /// lookahead session.
     func lookahead() -> Lookahead {
-      return Lookahead(lexemes: self.lexemes, currentToken: self.currentToken)
+      return Lookahead(
+        lexemes: self.lexemes,
+        currentToken: self.currentToken,
+        experimentalFeatures: self.experimentalFeatures
+      )
     }
   }
 

Sources/SwiftParser/ParseSourceFile.swift

@@ -22,6 +22,16 @@ extension Parser {
     return SourceFileSyntax.parse(from: &parser)
   }
 
+  /// A compiler-only interface that allows the enabling of experimental
+  /// features.
+  public static func parse(
+    source: UnsafeBufferPointer<UInt8>,
+    experimentalFeatures: ExperimentalFeatures
+  ) -> SourceFileSyntax {
+    var parser = Parser(source, experimentalFeatures: experimentalFeatures)
+    return SourceFileSyntax.parse(from: &parser)
+  }
+
   /// Parse the source code in the given buffer as Swift source file. See
   /// `Parser.init` for more details.
   public static func parse(

Sources/SwiftParser/Parser.swift

@@ -115,6 +115,9 @@ public struct Parser {
   /// Parser should own a ``LookaheadTracker`` so that we can share one `furthestOffset` in a parse.
   let lookaheadTrackerOwner = LookaheadTrackerOwner()
 
+  /// The experimental features that have been enabled.
+  let experimentalFeatures: ExperimentalFeatures
+
   /// A default maximum nesting level that is used if the client didn't
   /// explicitly specify one. Debug builds of the parser consume a lot more stack
   /// space and thus have a lower default maximum nesting level.
@@ -128,9 +131,11 @@ public struct Parser {
   public init(
     _ input: String,
     maximumNestingLevel: Int? = nil,
-    parseTransition: IncrementalParseTransition? = nil
+    parseTransition: IncrementalParseTransition? = nil,
+    experimentalFeatures: ExperimentalFeatures = []
   ) {
     self.maximumNestingLevel = maximumNestingLevel ?? Self.defaultMaximumNestingLevel
+    self.experimentalFeatures = experimentalFeatures
 
     self.arena = ParsingSyntaxArena(
       parseTriviaFunction: TriviaParser.parseTrivia(_:position:)
@@ -169,9 +174,11 @@ public struct Parser {
     _ input: UnsafeBufferPointer<UInt8>,
     maximumNestingLevel: Int? = nil,
     parseTransition: IncrementalParseTransition? = nil,
-    arena: ParsingSyntaxArena? = nil
+    arena: ParsingSyntaxArena? = nil,
+    experimentalFeatures: ExperimentalFeatures = []
   ) {
     self.maximumNestingLevel = maximumNestingLevel ?? Self.defaultMaximumNestingLevel
+    self.experimentalFeatures = experimentalFeatures
 
     var sourceBuffer: UnsafeBufferPointer<UInt8>
     if let arena {

Tests/SwiftParserTest/Assertions.swift

@@ -537,6 +537,7 @@ func assertParse(
   applyFixIts: [String]? = nil,
   fixedSource expectedFixedSource: String? = nil,
   options: AssertParseOptions = [],
+  experimentalFeatures: Parser.ExperimentalFeatures = [],
   file: StaticString = #file,
   line: UInt = #line
 ) {
@@ -549,6 +550,7 @@ func assertParse(
     applyFixIts: applyFixIts,
     fixedSource: expectedFixedSource,
     options: options,
+    experimentalFeatures: experimentalFeatures,
     file: file,
     line: line
   )
@@ -559,14 +561,15 @@ func assertParse(
 fileprivate func assertRoundTrip<S: SyntaxProtocol>(
   source: [UInt8],
   _ parse: (inout Parser) -> S,
+  experimentalFeatures: Parser.ExperimentalFeatures,
   file: StaticString,
   line: UInt
 ) {
   source.withUnsafeBufferPointer { buf in
     let mutatedSource = String(decoding: buf, as: UTF8.self)
     // Check that we don't hit any assertions in the parser while parsing
     // the mutated source and that it round-trips
-    var mutatedParser = Parser(buf)
+    var mutatedParser = Parser(buf, experimentalFeatures: experimentalFeatures)
     let mutatedTree = parse(&mutatedParser)
     // Run the diagnostic generator to make sure it doesn’t crash
     _ = ParseDiagnosticsGenerator.diagnostics(for: mutatedTree)
@@ -615,6 +618,7 @@ func assertParse<S: SyntaxProtocol>(
   applyFixIts: [String]? = nil,
   fixedSource expectedFixedSource: String? = nil,
   options: AssertParseOptions = [],
+  experimentalFeatures: Parser.ExperimentalFeatures = [],
   file: StaticString = #file,
   line: UInt = #line
 ) {
@@ -624,7 +628,7 @@ func assertParse<S: SyntaxProtocol>(
 
   let enableLongTests = ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] != "1"
 
-  var parser = Parser(source)
+  var parser = Parser(source, experimentalFeatures: experimentalFeatures)
   #if SWIFTPARSER_ENABLE_ALTERNATE_TOKEN_INTROSPECTION
   if enableLongTests {
     parser.enableAlternativeTokenChoices()
@@ -704,14 +708,14 @@ func assertParse<S: SyntaxProtocol>(
   }
 
   if expectedDiagnostics.isEmpty {
-    assertBasicFormat(source: source, parse: parse, file: file, line: line)
+    assertBasicFormat(source: source, parse: parse, experimentalFeatures: experimentalFeatures, file: file, line: line)
   }
 
   if enableLongTests {
     DispatchQueue.concurrentPerform(iterations: Array(tree.tokens(viewMode: .all)).count) { tokenIndex in
       let flippedTokenTree = TokenPresenceFlipper(flipTokenAtIndex: tokenIndex).rewrite(Syntax(tree))
       _ = ParseDiagnosticsGenerator.diagnostics(for: flippedTokenTree)
-      assertRoundTrip(source: flippedTokenTree.syntaxTextBytes, parse, file: file, line: line)
+      assertRoundTrip(source: flippedTokenTree.syntaxTextBytes, parse, experimentalFeatures: experimentalFeatures, file: file, line: line)
     }
 
     #if SWIFTPARSER_ENABLE_ALTERNATE_TOKEN_INTROSPECTION
@@ -721,7 +725,7 @@ func assertParse<S: SyntaxProtocol>(
     DispatchQueue.concurrentPerform(iterations: mutations.count) { index in
       let mutation = mutations[index]
       let alternateSource = MutatedTreePrinter.print(tree: Syntax(tree), mutations: [mutation.offset: mutation.replacement])
-      assertRoundTrip(source: alternateSource, parse, file: file, line: line)
+      assertRoundTrip(source: alternateSource, parse, experimentalFeatures: experimentalFeatures, file: file, line: line)
     }
     #endif
   }
@@ -748,15 +752,16 @@ class TriviaRemover: SyntaxRewriter {
 func assertBasicFormat<S: SyntaxProtocol>(
   source: String,
   parse: (inout Parser) -> S,
+  experimentalFeatures: Parser.ExperimentalFeatures,
   file: StaticString = #file,
   line: UInt = #line
 ) {
-  var parser = Parser(source)
+  var parser = Parser(source, experimentalFeatures: experimentalFeatures)
   let sourceTree = parse(&parser)
   let withoutTrivia = TriviaRemover(viewMode: .sourceAccurate).rewrite(sourceTree)
   let formatted = withoutTrivia.formatted()
 
-  var formattedParser = Parser(formatted.description)
+  var formattedParser = Parser(formatted.description, experimentalFeatures: experimentalFeatures)
   let formattedReparsed = Syntax(parse(&formattedParser))
 
   do {