WIP: Port incremental parsing ability to CodeBlockItem

Author: StevenWong12

Sources/SwiftParser/Lexer/LexemeSequence.swift

@@ -50,6 +50,19 @@ extension Lexer {
       return self.nextToken
     }
 
+    func getOffsetToStart(_ token: Lexer.Lexeme) -> Int {
+      return self.sourceBufferStart.distance(to: token.cursor)
+    }
+
+    mutating func advance(by offset: Int, currentToken: inout Lexer.Lexeme) {
+      self.cursor = currentToken.cursor
+      self.cursor.position = self.cursor.position.advanced(by: offset)
+
+      self.nextToken = self.cursor.nextToken(sourceBufferStart: self.sourceBufferStart, stateAllocator: lexerStateAllocator)
+
+      currentToken = self.advance()
+    }
+
     /// Reset the lexeme sequence to the state we were in when lexing `splitToken`
     /// but after we consumed `consumedPrefix` bytes from `splitToken`.
     /// - Warning: Do not add more usages of this function.

Sources/SwiftParser/Parser.swift

@@ -101,6 +101,8 @@ public struct Parser {
   /// When this nesting level is exceeded, the parser should stop parsing.
   let maximumNestingLevel: Int
 
+  let parseTransition: IncrementalParseTransition?
+
   /// A default maximum nesting level that is used if the client didn't
   /// explicitly specify one. Debug builds of the parser comume a lot more stack
   /// space and thus have a lower default maximum nesting level.
@@ -111,7 +113,7 @@ public struct Parser {
   #endif
 
   /// Initializes a ``Parser`` from the given string.
-  public init(_ input: String, maximumNestingLevel: Int? = nil) {
+  public init(_ input: String, maximumNestingLevel: Int? = nil, parseTransition: IncrementalParseTransition? = nil) {
     self.maximumNestingLevel = maximumNestingLevel ?? Self.defaultMaximumNestingLevel
 
     self.arena = ParsingSyntaxArena(
@@ -124,6 +126,7 @@ public struct Parser {
       return arena.internSourceBuffer(buffer)
     }
 
+    self.parseTransition = parseTransition
     self.lexemes = Lexer.tokenize(interned)
     self.currentToken = self.lexemes.advance()
   }
@@ -142,7 +145,7 @@ public struct Parser {
   ///            arena is created automatically, and `input` copied into the
   ///            arena. If non-`nil`, `input` must be within its registered
   ///            source buffer or allocator.
-  public init(_ input: UnsafeBufferPointer<UInt8>, maximumNestingLevel: Int? = nil, arena: ParsingSyntaxArena? = nil) {
+  public init(_ input: UnsafeBufferPointer<UInt8>, maximumNestingLevel: Int? = nil, arena: ParsingSyntaxArena? = nil, parseTransition: IncrementalParseTransition? = nil) {
     self.maximumNestingLevel = maximumNestingLevel ?? Self.defaultMaximumNestingLevel
 
     var sourceBuffer: UnsafeBufferPointer<UInt8>
@@ -157,6 +160,7 @@ public struct Parser {
       sourceBuffer = self.arena.internSourceBuffer(input)
     }
 
+    self.parseTransition = parseTransition
     self.lexemes = Lexer.tokenize(sourceBuffer)
     self.currentToken = self.lexemes.advance()
   }
@@ -621,3 +625,19 @@ extension Parser {
     )
   }
 }
+
+// MARK: Incremental Parsing
+extension Parser {
+  mutating func loadCurrentSyntaxNodeFromCache(for kind: SyntaxKind) -> Syntax? {
+    guard let parseTransition = self.parseTransition else { return nil }
+
+    var lookUpHelper = IncrementalParseLookup(transition: parseTransition)
+    let currentOffset = self.lexemes.getOffsetToStart(self.currentToken)
+    if let node = lookUpHelper.lookUp(currentOffset, kind: kind) {
+      self.lexemes.advance(by: node.byteSize, currentToken: &self.currentToken)
+      return node
+    }
+
+    return nil
+  }
+}

Sources/SwiftParser/TopLevel.swift

@@ -151,6 +151,10 @@ extension Parser {
   ///     statement → compiler-control-statement
   ///     statements → statement statements?
   mutating func parseCodeBlockItem(isAtTopLevel: Bool, allowInitDecl: Bool) -> RawCodeBlockItemSyntax? {
+    if let syntax = self.loadCurrentSyntaxNodeFromCache(for: .codeBlockItem) {
+      return RawCodeBlockItemSyntax(syntax.raw)
+    }
+
     if let remainingTokens = remainingTokensIfMaximumNestingLevelReached() {
       return RawCodeBlockItemSyntax(
         remainingTokens,

Sources/SwiftParser/generated/Parser+Entry.swift

@@ -21,7 +21,7 @@ extension Parser {
     source: String,
     parseTransition: IncrementalParseTransition? = nil
   ) -> SourceFileSyntax {
-    var parser = Parser(source)
+    var parser = Parser(source, parseTransition: parseTransition)
     return SourceFileSyntax.parse(from: &parser)
   }
 
@@ -32,7 +32,7 @@ extension Parser {
     maximumNestingLevel: Int? = nil,
     parseTransition: IncrementalParseTransition? = nil
   ) -> SourceFileSyntax {
-    var parser = Parser(source, maximumNestingLevel: maximumNestingLevel)
+    var parser = Parser(source, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition)
     return SourceFileSyntax.parse(from: &parser)
   }
 }

Sources/SwiftSyntax/IncrementalParseTransition.swift

@@ -242,7 +242,7 @@ public struct IncrementalParseLookup {
     let node = cursorLookup(prevPosition: prevPosition, kind: kind)
     if let delegate = reusedDelegate, let node {
       delegate.parserReusedNode(
-        range: ByteSourceRange(offset: newOffset, length: node.byteSize),
+        range: ByteSourceRange(offset: newOffset, length: node.byteSizeAfterTrimmingTrivia),
         previousNode: node
       )
     }