diff --git a/Documentation/LSP Extensions.md b/Documentation/LSP Extensions.md
index bdfde470c..695df0afa 100644
--- a/Documentation/LSP Extensions.md	
+++ b/Documentation/LSP Extensions.md	
@@ -35,8 +35,35 @@ Added field (this is an extension from clangd that SourceKit-LSP re-exposes):
 codeActions: CodeAction[]?
 ```
 
+## Semantic token modifiers
+
+Added the following cases from clangd
+
+```ts
+deduced = 'deduced'
+virtual = 'virtual'
+dependentName = 'dependentName'
+usedAsMutableReference = 'usedAsMutableReference'
+usedAsMutablePointer = 'usedAsMutablePointer'
+constructorOrDestructor = 'constructorOrDestructor'
+userDefined = 'userDefined'
+functionScope = 'functionScope'
+classScope = 'classScope'
+fileScope = 'fileScope'
+globalScope = 'globalScope'
+```
+
 ## Semantic token types
 
+Added the following cases from clangd
+
+```ts
+bracket = 'bracket'
+label = 'label'
+concept = 'concept'
+unknown = 'unknown'
+```
+
 Added case
 
 ```ts
diff --git a/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenModifiers.swift b/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenModifiers.swift
index d2e98055c..b623e52d1 100644
--- a/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenModifiers.swift
+++ b/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenModifiers.swift
@@ -34,6 +34,19 @@ public struct SemanticTokenModifiers: OptionSet, Hashable, Sendable {
   public static let documentation = Self(rawValue: 1 << 8)
   public static let defaultLibrary = Self(rawValue: 1 << 9)
 
+  // The following are LSP extensions from clangd
+  public static let deduced = Self(rawValue: 1 << 10)
+  public static let virtual = Self(rawValue: 1 << 11)
+  public static let dependentName = Self(rawValue: 1 << 12)
+  public static let usedAsMutableReference = Self(rawValue: 1 << 13)
+  public static let usedAsMutablePointer = Self(rawValue: 1 << 14)
+  public static let constructorOrDestructor = Self(rawValue: 1 << 15)
+  public static let userDefined = Self(rawValue: 1 << 16)
+  public static let functionScope = Self(rawValue: 1 << 17)
+  public static let classScope = Self(rawValue: 1 << 18)
+  public static let fileScope = Self(rawValue: 1 << 19)
+  public static let globalScope = Self(rawValue: 1 << 20)
+
   public var name: String? {
     switch self {
     case .declaration: return "declaration"
@@ -46,13 +59,24 @@ public struct SemanticTokenModifiers: OptionSet, Hashable, Sendable {
     case .modification: return "modification"
     case .documentation: return "documentation"
     case .defaultLibrary: return "defaultLibrary"
+    case .deduced: return "deduced"
+    case .virtual: return "virtual"
+    case .dependentName: return "dependentName"
+    case .usedAsMutableReference: return "usedAsMutableReference"
+    case .usedAsMutablePointer: return "usedAsMutablePointer"
+    case .constructorOrDestructor: return "constructorOrDestructor"
+    case .userDefined: return "userDefined"
+    case .functionScope: return "functionScope"
+    case .classScope: return "classScope"
+    case .fileScope: return "fileScope"
+    case .globalScope: return "globalScope"
     default: return nil
     }
   }
 
   /// All available modifiers, in ascending order of the bit index
   /// they are represented with (starting at the rightmost bit).
-  public static let predefined: [Self] = [
+  public static let all: [Self] = [
     .declaration,
     .definition,
     .readonly,
@@ -63,5 +87,16 @@ public struct SemanticTokenModifiers: OptionSet, Hashable, Sendable {
     .modification,
     .documentation,
     .defaultLibrary,
+    .deduced,
+    .virtual,
+    .dependentName,
+    .usedAsMutableReference,
+    .usedAsMutablePointer,
+    .constructorOrDestructor,
+    .userDefined,
+    .functionScope,
+    .classScope,
+    .fileScope,
+    .globalScope,
   ]
 }
diff --git a/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenTypes.swift b/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenTypes.swift
index 57fe5d471..0deafb5a8 100644
--- a/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenTypes.swift
+++ b/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenTypes.swift
@@ -50,7 +50,18 @@ public struct SemanticTokenTypes: Hashable, Sendable {
   /// since 3.17.0
   public static let decorator = Self("decorator")
 
-  public static let predefined: [Self] = [
+  // The following are LSP extensions from clangd
+  public static let bracket = Self("bracket")
+  public static let label = Self("label")
+  public static let concept = Self("concept")
+  public static let unknown = Self("unknown")
+
+  /// An identifier that hasn't been further classified
+  ///
+  /// **(LSP Extension)**
+  public static let identifier = Self("identifier")
+
+  public static let all: [Self] = [
     .namespace,
     .type,
     .class,
@@ -73,5 +84,11 @@ public struct SemanticTokenTypes: Hashable, Sendable {
     .number,
     .regexp,
     .operator,
+    .decorator,
+    .bracket,
+    .label,
+    .concept,
+    .unknown,
+    .identifier,
   ]
 }
diff --git a/Sources/SourceKitLSP/CMakeLists.txt b/Sources/SourceKitLSP/CMakeLists.txt
index 0f7b6fc43..6d8f2a692 100644
--- a/Sources/SourceKitLSP/CMakeLists.txt
+++ b/Sources/SourceKitLSP/CMakeLists.txt
@@ -11,6 +11,7 @@ add_library(SourceKitLSP STATIC
   MessageHandlingDependencyTracker.swift
   Rename.swift
   ResponseError+Init.swift
+  SemanticTokensLegend+SourceKitLSPLegend.swift
   SourceKitIndexDelegate.swift
   SourceKitLSPCommandMetadata.swift
   SourceKitLSPServer.swift
@@ -22,7 +23,9 @@ add_library(SourceKitLSP STATIC
   Workspace.swift
 )
 target_sources(SourceKitLSP PRIVATE
-  Clang/ClangLanguageService.swift)
+  Clang/ClangLanguageService.swift
+  Clang/SemanticTokenTranslator.swift
+)
 target_sources(SourceKitLSP PRIVATE
   Swift/AdjustPositionToStartOfIdentifier.swift
   Swift/CodeActions/AddDocumentation.swift
diff --git a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift
index ccdeecb9b..0ec42ee66 100644
--- a/Sources/SourceKitLSP/Clang/ClangLanguageService.swift
+++ b/Sources/SourceKitLSP/Clang/ClangLanguageService.swift
@@ -95,6 +95,9 @@ actor ClangLanguageService: LanguageService, MessageHandler {
   /// opened with.
   private var openDocuments: [DocumentURI: Language] = [:]
 
+  /// Type to map `clangd`'s semantic token legend to SourceKit-LSP's.
+  private var semanticTokensTranslator: SemanticTokensLegendTranslator? = nil
+
   /// While `clangd` is running, its PID.
   #if os(Windows)
   private var hClangd: HANDLE = INVALID_HANDLE_VALUE
@@ -412,6 +415,12 @@ extension ClangLanguageService {
 
     let result = try await clangd.send(initialize)
     self.capabilities = result.capabilities
+    if let legend = result.capabilities.semanticTokensProvider?.legend {
+      self.semanticTokensTranslator = SemanticTokensLegendTranslator(
+        clangdLegend: legend,
+        sourceKitLSPLegend: SemanticTokensLegend.sourceKitLSPLegend
+      )
+    }
     return result
   }
 
@@ -537,19 +546,50 @@ extension ClangLanguageService {
   }
 
   func documentSemanticTokens(_ req: DocumentSemanticTokensRequest) async throws -> DocumentSemanticTokensResponse? {
-    return try await forwardRequestToClangd(req)
+    guard var response = try await forwardRequestToClangd(req) else {
+      return nil
+    }
+    if let semanticTokensTranslator {
+      response.data = semanticTokensTranslator.translate(response.data)
+    }
+    return response
   }
 
   func documentSemanticTokensDelta(
     _ req: DocumentSemanticTokensDeltaRequest
   ) async throws -> DocumentSemanticTokensDeltaResponse? {
-    return try await forwardRequestToClangd(req)
+    guard var response = try await forwardRequestToClangd(req) else {
+      return nil
+    }
+    if let semanticTokensTranslator {
+      switch response {
+      case .tokens(var tokens):
+        tokens.data = semanticTokensTranslator.translate(tokens.data)
+        response = .tokens(tokens)
+      case .delta(var delta):
+        delta.edits = delta.edits.map {
+          var edit = $0
+          if let data = edit.data {
+            edit.data = semanticTokensTranslator.translate(data)
+          }
+          return edit
+        }
+        response = .delta(delta)
+      }
+    }
+    return response
   }
 
   func documentSemanticTokensRange(
     _ req: DocumentSemanticTokensRangeRequest
   ) async throws -> DocumentSemanticTokensResponse? {
-    return try await forwardRequestToClangd(req)
+    guard var response = try await forwardRequestToClangd(req) else {
+      return nil
+    }
+    if let semanticTokensTranslator {
+      response.data = semanticTokensTranslator.translate(response.data)
+    }
+    return response
   }
 
   func colorPresentation(_ req: ColorPresentationRequest) async throws -> [ColorPresentation] {
diff --git a/Sources/SourceKitLSP/Clang/SemanticTokenTranslator.swift b/Sources/SourceKitLSP/Clang/SemanticTokenTranslator.swift
new file mode 100644
index 000000000..f4e4b7767
--- /dev/null
+++ b/Sources/SourceKitLSP/Clang/SemanticTokenTranslator.swift
@@ -0,0 +1,136 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import LSPLogging
+import LanguageServerProtocol
+
+/// `clangd` might use a different semantic token legend than SourceKit-LSP.
+///
+/// This type allows translation the semantic tokens from `clangd` into the token legend that is used by SourceKit-LSP.
+struct SemanticTokensLegendTranslator {
+  private enum Translation {
+    /// The token type or modifier from clangd does not exist in SourceKit-LSP
+    case doesNotExistInSourceKitLSP
+
+    /// The token type or modifier exists in SourceKit-LSP but it uses a different index. We need to translate the
+    /// clangd index to this SourceKit-LSP index.
+    case translation(UInt32)
+  }
+
+  /// For all token types whose representation in clang differs from the representation in SourceKit-LSP, maps the
+  /// index of that token type in clangd’s token type legend to the corresponding representation in SourceKit-LSP.
+  private let tokenTypeTranslations: [UInt32: Translation]
+
+  /// For all token modifiers whose representation in clang differs from the representation in SourceKit-LSP, maps the
+  /// index of that token modifier in clangd’s token type legend to the corresponding representation in SourceKit-LSP.
+  private let tokenModifierTranslations: [UInt32: Translation]
+
+  /// A bitmask that has all bits set to 1 that are used for clangd token modifiers which have a different
+  /// representation in SourceKit-LSP. If a token modifier does not have any bits set in common with this bitmask, no
+  /// token mapping needs to be performed.
+  private let tokenModifierTranslationBitmask: UInt32
+
+  /// For token types in clangd that do not exist in SourceKit-LSP's token legend, we need to map their token types to
+  /// some valid SourceKit-LSP token type. Use the token type with this index.
+  private let tokenTypeFallbackIndex: UInt32
+
+  init(clangdLegend: SemanticTokensLegend, sourceKitLSPLegend: SemanticTokensLegend) {
+    var tokenTypeTranslations: [UInt32: Translation] = [:]
+    for (index, tokenType) in clangdLegend.tokenTypes.enumerated() {
+      switch sourceKitLSPLegend.tokenTypes.firstIndex(of: tokenType) {
+      case index:
+        break
+      case nil:
+        logger.error("Token type '\(tokenType, privacy: .public)' from clangd does not exist in SourceKit-LSP's legend")
+        tokenTypeTranslations[UInt32(index)] = .doesNotExistInSourceKitLSP
+      case let sourceKitLSPIndex?:
+        logger.info(
+          "Token type '\(tokenType, privacy: .public)' from clangd at index \(index) translated to \(sourceKitLSPIndex)"
+        )
+        tokenTypeTranslations[UInt32(index)] = .translation(UInt32(sourceKitLSPIndex))
+      }
+    }
+    self.tokenTypeTranslations = tokenTypeTranslations
+
+    var tokenModifierTranslations: [UInt32: Translation] = [:]
+    for (index, tokenModifier) in clangdLegend.tokenModifiers.enumerated() {
+      switch sourceKitLSPLegend.tokenModifiers.firstIndex(of: tokenModifier) {
+      case index:
+        break
+      case nil:
+        logger.error(
+          "Token modifier '\(tokenModifier, privacy: .public)' from clangd does not exist in SourceKit-LSP's legend"
+        )
+        tokenModifierTranslations[UInt32(index)] = .doesNotExistInSourceKitLSP
+      case let sourceKitLSPIndex?:
+        logger.error(
+          "Token modifier '\(tokenModifier, privacy: .public)' from clangd at index \(index) translated to \(sourceKitLSPIndex)"
+        )
+        tokenModifierTranslations[UInt32(index)] = .translation(UInt32(sourceKitLSPIndex))
+      }
+    }
+    self.tokenModifierTranslations = tokenModifierTranslations
+
+    var tokenModifierTranslationBitmask: UInt32 = 0
+    for translatedIndex in tokenModifierTranslations.keys {
+      tokenModifierTranslationBitmask.setBitToOne(at: Int(translatedIndex))
+    }
+    self.tokenModifierTranslationBitmask = tokenModifierTranslationBitmask
+
+    self.tokenTypeFallbackIndex = UInt32(
+      sourceKitLSPLegend.tokenTypes.firstIndex(of: SemanticTokenTypes.unknown.name) ?? 0
+    )
+  }
+
+  func translate(_ data: [UInt32]) -> [UInt32] {
+    var data = data
+    // Translate token types, which are at offset n + 3.
+    for i in stride(from: 3, to: data.count, by: 5) {
+      switch tokenTypeTranslations[data[i]] {
+      case .doesNotExistInSourceKitLSP: data[i] = tokenTypeFallbackIndex
+      case .translation(let translatedIndex): data[i] = translatedIndex
+      case nil: break
+      }
+    }
+
+    // Translate token modifiers, which are at offset n + 4
+    for i in stride(from: 4, to: data.count, by: 5) {
+      guard data[i] & tokenModifierTranslationBitmask != 0 else {
+        // Fast path: There is nothing to translate
+        continue
+      }
+      var translatedModifiersBitmask: UInt32 = 0
+      for (clangdModifier, sourceKitLSPModifier) in tokenModifierTranslations {
+        guard data[i].hasBitSet(at: Int(clangdModifier)) else {
+          continue
+        }
+        switch sourceKitLSPModifier {
+        case .doesNotExistInSourceKitLSP: break
+        case .translation(let sourceKitLSPIndex): translatedModifiersBitmask.setBitToOne(at: Int(sourceKitLSPIndex))
+        }
+      }
+      data[i] = data[i] & ~tokenModifierTranslationBitmask | translatedModifiersBitmask
+    }
+
+    return data
+  }
+}
+
+fileprivate extension UInt32 {
+  mutating func hasBitSet(at index: Int) -> Bool {
+    return self & (1 << index) != 0
+  }
+
+  mutating func setBitToOne(at index: Int) {
+    self |= 1 << index
+  }
+}
diff --git a/Sources/SourceKitLSP/SemanticTokensLegend+SourceKitLSPLegend.swift b/Sources/SourceKitLSP/SemanticTokensLegend+SourceKitLSPLegend.swift
new file mode 100644
index 000000000..833c18e5d
--- /dev/null
+++ b/Sources/SourceKitLSP/SemanticTokensLegend+SourceKitLSPLegend.swift
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import LanguageServerProtocol
+
+extension SemanticTokenTypes {
+  // LSP doesn’t know about actors. Display actors as classes.
+  public static var actor: Self { Self.class }
+
+  /// Token types are looked up by index
+  public var tokenType: UInt32 {
+    UInt32(Self.all.firstIndex(of: self)!)
+  }
+}
+
+extension SemanticTokensLegend {
+  /// The semantic tokens legend that is used between SourceKit-LSP and the editor.
+  static let sourceKitLSPLegend = SemanticTokensLegend(
+    tokenTypes: SemanticTokenTypes.all.map(\.name),
+    tokenModifiers: SemanticTokenModifiers.all.compactMap(\.name)
+  )
+}
diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift
index 05aad8ee8..6984aa4a9 100644
--- a/Sources/SourceKitLSP/SourceKitLSPServer.swift
+++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift
@@ -1094,10 +1094,7 @@ extension SourceKitLSPServer {
       await registry.clientHasDynamicSemanticTokensRegistration
       ? nil
       : SemanticTokensOptions(
-        legend: SemanticTokensLegend(
-          tokenTypes: SemanticTokenTypes.all.map(\.name),
-          tokenModifiers: SemanticTokenModifiers.all.compactMap(\.name)
-        ),
+        legend: SemanticTokensLegend.sourceKitLSPLegend,
         range: .bool(true),
         full: .bool(true)
       )
diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift
index a020d4d25..96cb2f1e6 100644
--- a/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift
+++ b/Sources/SourceKitLSP/Swift/SwiftLanguageService.swift
@@ -292,10 +292,7 @@ extension SwiftLanguageService {
           commands: builtinSwiftCommands
         ),
         semanticTokensProvider: SemanticTokensOptions(
-          legend: SemanticTokensLegend(
-            tokenTypes: SemanticTokenTypes.all.map(\.name),
-            tokenModifiers: SemanticTokenModifiers.all.compactMap(\.name)
-          ),
+          legend: SemanticTokensLegend.sourceKitLSPLegend,
           range: .bool(true),
           full: .bool(true)
         ),
diff --git a/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift b/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift
index db2402917..b17f9f538 100644
--- a/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift
+++ b/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift
@@ -47,24 +47,3 @@ public struct SyntaxHighlightingToken: Hashable, Sendable {
     self.init(range: range, kind: kind, modifiers: modifiers)
   }
 }
-
-extension SemanticTokenTypes {
-  /// **(LSP Extension)**
-  public static let identifier = Self("identifier")
-
-  // LSP doesn’t know about actors. Display actors as classes.
-  public static let actor = Self("class")
-
-  /// All tokens supported by sourcekit-lsp
-  public static let all: [Self] = predefined + [.identifier, .actor]
-
-  /// Token types are looked up by index
-  public var tokenType: UInt32 {
-    UInt32(Self.all.firstIndex(of: self)!)
-  }
-}
-
-extension SemanticTokenModifiers {
-  /// All tokens supported by sourcekit-lsp
-  public static let all: [Self] = predefined
-}
diff --git a/Tests/SourceKitLSPTests/SemanticTokensTests.swift b/Tests/SourceKitLSPTests/SemanticTokensTests.swift
index 32097d7bf..ae8292add 100644
--- a/Tests/SourceKitLSPTests/SemanticTokensTests.swift
+++ b/Tests/SourceKitLSPTests/SemanticTokensTests.swift
@@ -12,6 +12,7 @@
 
 import LSPTestSupport
 import LanguageServerProtocol
+import SKSupport
 import SKTestSupport
 import SourceKitD
 @_spi(Testing) import SourceKitLSP
@@ -20,144 +21,6 @@ import XCTest
 private typealias Token = SyntaxHighlightingToken
 
 final class SemanticTokensTests: XCTestCase {
-  /// The mock client used to communicate with the SourceKit-LSP server.
-  ///
-  /// - Note: Set before each test run in `setUp`.
-  private var testClient: TestSourceKitLSPClient! = nil
-
-  /// The URI of the document that is being tested by the current test case.
-  ///
-  /// - Note: This URI is set to a unique value before each test case in `setUp`.
-  private var uri: DocumentURI!
-
-  /// The current version of the document being opened.
-  ///
-  /// - Note: This gets reset to 0 in `setUp` and incremented on every call to
-  ///   `openDocument` and `editDocument`.
-  private var version: Int = 0
-
-  override func setUp() async throws {
-    version = 0
-    uri = DocumentURI(URL(fileURLWithPath: "/SemanticTokensTests/\(UUID()).swift"))
-    testClient = try await TestSourceKitLSPClient(
-      capabilities: ClientCapabilities(
-        workspace: .init(
-          semanticTokens: .init(
-            refreshSupport: true
-          )
-        ),
-        textDocument: .init(
-          semanticTokens: .init(
-            dynamicRegistration: true,
-            requests: .init(
-              range: .bool(true),
-              full: .bool(true)
-            ),
-            tokenTypes: SemanticTokenTypes.all.map(\.name),
-            tokenModifiers: SemanticTokenModifiers.all.compactMap(\.name),
-            formats: [.relative]
-          )
-        )
-      ),
-      usePullDiagnostics: false
-    )
-  }
-
-  override func tearDown() {
-    testClient = nil
-  }
-
-  private func openDocument(text: String) {
-    // We will wait for the server to dynamically register semantic tokens
-
-    let registerCapabilityExpectation = expectation(description: "\(#function) - register semantic tokens capability")
-    testClient.handleSingleRequest { (req: RegisterCapabilityRequest) -> VoidResponse in
-      let capabilityRegistration = req.registrations.first { reg in
-        reg.method == SemanticTokensRegistrationOptions.method
-      }
-
-      guard case .dictionary(let registerOptionsDict) = capabilityRegistration?.registerOptions,
-        let registerOptions = SemanticTokensRegistrationOptions(fromLSPDictionary: registerOptionsDict)
-      else {
-        XCTFail("Expected semantic tokens registration options dictionary")
-        return VoidResponse()
-      }
-
-      XCTAssertFalse(
-        registerOptions.semanticTokenOptions.legend.tokenTypes.isEmpty,
-        "Expected semantic tokens legend"
-      )
-
-      registerCapabilityExpectation.fulfill()
-      return VoidResponse()
-    }
-
-    // We will wait for the first refresh request to make sure that the semantic tokens are ready
-
-    testClient.openDocument(text, uri: uri)
-    version += 1
-
-    wait(for: [registerCapabilityExpectation], timeout: defaultTimeout)
-  }
-
-  private func editDocument(changes: [TextDocumentContentChangeEvent], expectRefresh: Bool = true) {
-    // We wait for the semantic tokens again
-    // Note that we assume to already have called openDocument before
-
-    testClient.send(
-      DidChangeTextDocumentNotification(
-        textDocument: VersionedTextDocumentIdentifier(
-          uri,
-          version: version
-        ),
-        contentChanges: changes
-      )
-    )
-    version += 1
-  }
-
-  private func editDocument(range: Range<Position>, text: String, expectRefresh: Bool = true) {
-    editDocument(
-      changes: [
-        TextDocumentContentChangeEvent(
-          range: range,
-          text: text
-        )
-      ],
-      expectRefresh: expectRefresh
-    )
-  }
-
-  private func performSemanticTokensRequest(range: Range<Position>? = nil) async throws -> SyntaxHighlightingTokens {
-    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
-    let response: DocumentSemanticTokensResponse!
-
-    if let range = range {
-      response = try await testClient.send(
-        DocumentSemanticTokensRangeRequest(
-          textDocument: TextDocumentIdentifier(uri),
-          range: range
-        )
-      )
-    } else {
-      response = try await testClient.send(
-        DocumentSemanticTokensRequest(
-          textDocument: TextDocumentIdentifier(uri)
-        )
-      )
-    }
-
-    return SyntaxHighlightingTokens(lspEncodedTokens: response.data)
-  }
-
-  private func openAndPerformSemanticTokensRequest(
-    text: String,
-    range: Range<Position>? = nil
-  ) async throws -> SyntaxHighlightingTokens {
-    openDocument(text: text)
-    return try await performSemanticTokensRequest(range: range)
-  }
-
   func testIntArrayCoding() async throws {
     let tokens = SyntaxHighlightingTokens(tokens: [
       Token(
@@ -196,17 +59,21 @@ final class SemanticTokensTests: XCTestCase {
   }
 
   func testRangeSplitting() async throws {
-    let text = """
-      struct X {
-        let x: Int
-        let y: String
-
-
-      }
-      """
-    openDocument(text: text)
-
-    let snapshot = try await testClient.server.documentManager.latestSnapshot(uri)
+    let snapshot = DocumentSnapshot(
+      uri: DocumentURI(for: .swift),
+      language: .swift,
+      version: 0,
+      lineTable: LineTable(
+        """
+        struct X {
+          let x: Int
+          let y: String
+
+
+        }
+        """
+      )
+    )
 
     let empty = Position(line: 0, utf16index: 1)..<Position(line: 0, utf16index: 1)
     XCTAssertEqual(empty.splitToSingleLineRanges(in: snapshot), [])
@@ -231,673 +98,928 @@ final class SemanticTokensTests: XCTestCase {
   }
 
   func testEmpty() async throws {
-    let text = ""
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(tokens.tokens, [])
+    try await assertSemanticTokens(
+      markedContents: "",
+      expected: []
+    )
   }
 
   func testRanged() async throws {
-    let text = """
-      let x = 1
-      let test = 20
-      let abc = 333
-      let y = 4
-      """
-    let start = Position(line: 1, utf16index: 0)
-    let end = Position(line: 2, utf16index: 5)
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text, range: start..<end)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        Token(line: 1, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 1, utf16index: 4, length: 4, kind: .identifier),
-        Token(line: 1, utf16index: 11, length: 2, kind: .number),
-        Token(line: 2, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 2, utf16index: 4, length: 3, kind: .identifier),
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: """
+        let x = 1
+        1️⃣let 2️⃣test = 3️⃣20
+        4️⃣let 5️⃣a6️⃣bc = 333
+        let y = 4
+        """,
+      range: ("1️⃣", "6️⃣"),
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 4, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 2, kind: .number),
+        TokenSpec(marker: "4️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "5️⃣", length: 3, kind: .identifier),
       ]
     )
   }
 
   func testLexicalTokens() async throws {
-    let text = """
-      let x = 3
-      var y = "test"
-      /* abc */ // 123
-      """
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣let 2️⃣x = 3️⃣3
+        4️⃣var 5️⃣y = 6️⃣"test"
+        7️⃣/* abc */ 8️⃣// 123
+        """,
+      expected: [
         // let x = 3
-        Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 0, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 0, utf16index: 8, length: 1, kind: .number),
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 1, kind: .number),
         // var y = "test"
-        Token(line: 1, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 1, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 1, utf16index: 8, length: 6, kind: .string),
+        TokenSpec(marker: "4️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "5️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "6️⃣", length: 6, kind: .string),
         // /* abc */ // 123
-        Token(line: 2, utf16index: 0, length: 9, kind: .comment),
-        Token(line: 2, utf16index: 10, length: 6, kind: .comment),
+        TokenSpec(marker: "7️⃣", length: 9, kind: .comment),
+        TokenSpec(marker: "8️⃣", length: 6, kind: .comment),
       ]
     )
   }
 
   func testLexicalTokensForMultiLineComments() async throws {
-    let text = """
-      let x = 3 /*
-      let x = 12
-      */
-      """
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 0, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 0, utf16index: 8, length: 1, kind: .number),
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣let 2️⃣x = 3️⃣3 4️⃣/*
+        5️⃣let x = 12
+        6️⃣*/
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 1, kind: .number),
         // Multi-line comments are split into single-line tokens
-        Token(line: 0, utf16index: 10, length: 2, kind: .comment),
-        Token(line: 1, utf16index: 0, length: 10, kind: .comment),
-        Token(line: 2, utf16index: 0, length: 2, kind: .comment),
+        TokenSpec(marker: "4️⃣", length: 2, kind: .comment),
+        TokenSpec(marker: "5️⃣", length: 10, kind: .comment),
+        TokenSpec(marker: "6️⃣", length: 2, kind: .comment),
       ]
     )
   }
 
   func testLexicalTokensForDocComments() async throws {
-    let text = """
-      /** abc */
-        /// def
-      """
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 10, kind: .comment, modifiers: [.documentation]),
-        Token(line: 1, utf16index: 2, length: 7, kind: .comment, modifiers: [.documentation]),
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣/** abc */
+          2️⃣/// def
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 10, kind: .comment, modifiers: .documentation),
+        TokenSpec(marker: "2️⃣", length: 7, kind: .comment, modifiers: .documentation),
       ]
     )
   }
 
   func testLexicalTokensForBackticks() async throws {
-    let text = """
-      var `if` = 20
-      let `else` = 3
-      let `onLeft = ()
-      let onRight` = ()
-      """
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        // var `if` = 20
-        Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 0, utf16index: 4, length: 4, kind: .identifier),
-        Token(line: 0, utf16index: 11, length: 2, kind: .number),
-        // let `else` = 3
-        Token(line: 1, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 1, utf16index: 4, length: 6, kind: .identifier),
-        Token(line: 1, utf16index: 13, length: 1, kind: .number),
-        // let `onLeft = ()
-        Token(line: 2, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 2, utf16index: 5, length: 6, kind: .identifier),
-        // let onRight` = ()
-        Token(line: 3, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 3, utf16index: 4, length: 7, kind: .identifier),
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣var 2️⃣`if` = 3️⃣20
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 4, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 2, kind: .number),
+      ]
+    )
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣let 2️⃣`else` = 3️⃣3
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 6, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 1, kind: .number),
+      ]
+    )
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣let `2️⃣onLeft = ()
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 6, kind: .identifier),
+      ]
+    )
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣let 2️⃣onRight` = ()
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 7, kind: .identifier),
       ]
     )
   }
 
   func testSemanticTokens() async throws {
-    let text = """
-      struct X {}
-
-      let x = X()
-      let y = x + x
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
 
-      func a() {}
-      let b = {}
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣struct 2️⃣X {}
 
-      a()
-      b()
-      """
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
+        3️⃣let 4️⃣x = 5️⃣X()
+        6️⃣let 7️⃣y = 8️⃣x 9️⃣+ 🔟x
+        """,
+      expected: [
         // struct X {}
-        Token(line: 0, utf16index: 0, length: 6, kind: .keyword),
-        Token(line: 0, utf16index: 7, length: 1, kind: .identifier),
+        TokenSpec(marker: "1️⃣", length: 6, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
         // let x = X()
-        Token(line: 2, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 2, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 2, utf16index: 8, length: 1, kind: .struct),
+        TokenSpec(marker: "3️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "4️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "5️⃣", length: 1, kind: .struct),
         // let y = x + x
-        Token(line: 3, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 3, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 3, utf16index: 8, length: 1, kind: .variable),
-        Token(line: 3, utf16index: 10, length: 1, kind: .operator),
-        Token(line: 3, utf16index: 12, length: 1, kind: .variable),
+        TokenSpec(marker: "6️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "7️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "8️⃣", length: 1, kind: .variable),
+        TokenSpec(marker: "9️⃣", length: 1, kind: .operator),
+        TokenSpec(marker: "🔟", length: 1, kind: .variable),
+      ]
+    )
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣func 2️⃣a() {}
+        3️⃣let 4️⃣b = {}
+
+        5️⃣a()
+        6️⃣b()
+        """,
+      expected: [
         // func a() {}
-        Token(line: 5, utf16index: 0, length: 4, kind: .keyword),
-        Token(line: 5, utf16index: 5, length: 1, kind: .identifier),
+        TokenSpec(marker: "1️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
         // let b = {}
-        Token(line: 6, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 6, utf16index: 4, length: 1, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "4️⃣", length: 1, kind: .identifier),
         // a()
-        Token(line: 8, utf16index: 0, length: 1, kind: .function),
+        TokenSpec(marker: "5️⃣", length: 1, kind: .function),
         // b()
-        Token(line: 9, utf16index: 0, length: 1, kind: .variable),
+        TokenSpec(marker: "6️⃣", length: 1, kind: .variable),
       ]
     )
   }
 
   func testSemanticTokensForProtocols() async throws {
-    let text = """
-      protocol X {}
-      class Y: X {}
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
 
-      let y: Y = X()
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣protocol 2️⃣X {}
+        3️⃣class 4️⃣Y: 5️⃣X {}
 
-      func f<T: X>() {}
-      """
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
+        6️⃣let 7️⃣y: 8️⃣Y = 9️⃣X()
+        """,
+      expected: [
         // protocol X {}
-        Token(line: 0, utf16index: 0, length: 8, kind: .keyword),
-        Token(line: 0, utf16index: 9, length: 1, kind: .identifier),
+        TokenSpec(marker: "1️⃣", length: 8, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
         // class Y: X {}
-        Token(line: 1, utf16index: 0, length: 5, kind: .keyword),
-        Token(line: 1, utf16index: 6, length: 1, kind: .identifier),
-        Token(line: 1, utf16index: 9, length: 1, kind: .interface),
+        TokenSpec(marker: "3️⃣", length: 5, kind: .keyword),
+        TokenSpec(marker: "4️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "5️⃣", length: 1, kind: .interface),
         // let y: Y = X()
-        Token(line: 3, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 3, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 3, utf16index: 7, length: 1, kind: .class),
-        Token(line: 3, utf16index: 11, length: 1, kind: .interface),
+        TokenSpec(marker: "6️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "7️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "8️⃣", length: 1, kind: .class),
+        TokenSpec(marker: "9️⃣", length: 1, kind: .interface),
+      ]
+    )
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣protocol 2️⃣X {}
+
+        3️⃣func 4️⃣f<5️⃣T: 6️⃣X>() {}
+        """,
+      expected: [
+        // protocol X {}
+        TokenSpec(marker: "1️⃣", length: 8, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
         // func f<T: X>() {}
-        Token(line: 5, utf16index: 0, length: 4, kind: .keyword),
-        Token(line: 5, utf16index: 5, length: 1, kind: .identifier),
-        Token(line: 5, utf16index: 7, length: 1, kind: .identifier),
-        Token(line: 5, utf16index: 10, length: 1, kind: .interface),
+        TokenSpec(marker: "3️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "4️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "5️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "6️⃣", length: 1, kind: .interface),
       ]
     )
   }
 
   func testSemanticTokensForFunctionSignatures() async throws {
-    let text = "func f(x: Int, _ y: String) {}"
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 4, kind: .keyword),
-        Token(line: 0, utf16index: 5, length: 1, kind: .identifier),
-        Token(line: 0, utf16index: 7, length: 1, kind: .function),
-        Token(line: 0, utf16index: 10, length: 3, kind: .struct, modifiers: .defaultLibrary),
-        Token(line: 0, utf16index: 17, length: 1, kind: .identifier),
-        Token(line: 0, utf16index: 20, length: 6, kind: .struct, modifiers: .defaultLibrary),
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: "1️⃣func 2️⃣f(3️⃣x: 4️⃣Int, _ 5️⃣y: 6️⃣String) {}",
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 1, kind: .function),
+        TokenSpec(marker: "4️⃣", length: 3, kind: .struct, modifiers: .defaultLibrary),
+        TokenSpec(marker: "5️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "6️⃣", length: 6, kind: .struct, modifiers: .defaultLibrary),
       ]
     )
   }
 
   func testSemanticTokensForFunctionSignaturesWithEmoji() async throws {
-    let text = "func x👍y() {}"
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 4, kind: .keyword),
-        Token(line: 0, utf16index: 5, length: 4, kind: .identifier),
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: "1️⃣func 2️⃣x👍y() {}",
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 4, kind: .identifier),
       ]
     )
   }
 
   func testSemanticTokensForStaticMethods() async throws {
-    let text = """
-      class X {
-        deinit {}
-        static func f() {}
-        class func g() {}
-      }
-      X.f()
-      X.g()
-      """
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣class 2️⃣X {
+          3️⃣static 4️⃣func 5️⃣f() {}
+        }
+        6️⃣X.7️⃣f()
+        """,
+      expected: [
         // class X
-        Token(line: 0, utf16index: 0, length: 5, kind: .keyword),
-        Token(line: 0, utf16index: 6, length: 1, kind: .identifier),
-        // deinit {}
-        Token(line: 1, utf16index: 2, length: 6, kind: .keyword),
+        TokenSpec(marker: "1️⃣", length: 5, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
         // static func f() {}
-        Token(line: 2, utf16index: 2, length: 6, kind: .keyword),
-        Token(line: 2, utf16index: 9, length: 4, kind: .keyword),
-        Token(line: 2, utf16index: 14, length: 1, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 6, kind: .keyword),
+        TokenSpec(marker: "4️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "5️⃣", length: 1, kind: .identifier),
+        // X.f()
+        TokenSpec(marker: "6️⃣", length: 1, kind: .class),
+        TokenSpec(marker: "7️⃣", length: 1, kind: .method, modifiers: .static),
+      ]
+    )
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣class 2️⃣X {
+          3️⃣class 4️⃣func 5️⃣g() {}
+        }
+        6️⃣X.7️⃣g()
+        """,
+      expected: [
+        // class X
+        TokenSpec(marker: "1️⃣", length: 5, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
         // class func g() {}
-        Token(line: 3, utf16index: 2, length: 5, kind: .keyword),
-        Token(line: 3, utf16index: 8, length: 4, kind: .keyword),
-        Token(line: 3, utf16index: 13, length: 1, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 5, kind: .keyword),
+        TokenSpec(marker: "4️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "5️⃣", length: 1, kind: .identifier),
         // X.f()
-        Token(line: 5, utf16index: 0, length: 1, kind: .class),
-        Token(line: 5, utf16index: 2, length: 1, kind: .method, modifiers: [.static]),
-        // X.g()
-        Token(line: 6, utf16index: 0, length: 1, kind: .class),
-        Token(line: 6, utf16index: 2, length: 1, kind: .method, modifiers: [.static]),
+        TokenSpec(marker: "6️⃣", length: 1, kind: .class),
+        TokenSpec(marker: "7️⃣", length: 1, kind: .method, modifiers: .static),
       ]
     )
   }
 
   func testSemanticTokensForEnumMembers() async throws {
-    let text = """
-      enum Maybe<T> {
-        case none
-        case some(T)
-      }
-
-      let x = Maybe<String>.none
-      let y: Maybe = .some(42)
-      """
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣enum 2️⃣Maybe<3️⃣T> {
+          4️⃣case 5️⃣none
+        }
+
+        6️⃣let 7️⃣x = 8️⃣Maybe<9️⃣String>.🔟none
+        """,
+      expected: [
         // enum Maybe<T>
-        Token(line: 0, utf16index: 0, length: 4, kind: .keyword),
-        Token(line: 0, utf16index: 5, length: 5, kind: .identifier),
-        Token(line: 0, utf16index: 11, length: 1, kind: .identifier),
+        TokenSpec(marker: "1️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 5, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 1, kind: .identifier),
         // case none
-        Token(line: 1, utf16index: 2, length: 4, kind: .keyword),
-        Token(line: 1, utf16index: 7, length: 4, kind: .identifier),
-        // case some
-        Token(line: 2, utf16index: 2, length: 4, kind: .keyword),
-        Token(line: 2, utf16index: 7, length: 4, kind: .identifier),
-        Token(line: 2, utf16index: 12, length: 1, kind: .typeParameter),
+        TokenSpec(marker: "4️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "5️⃣", length: 4, kind: .identifier),
         // let x = Maybe<String>.none
-        Token(line: 5, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 5, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 5, utf16index: 8, length: 5, kind: .enum),
-        Token(line: 5, utf16index: 14, length: 6, kind: .struct, modifiers: .defaultLibrary),
-        Token(line: 5, utf16index: 22, length: 4, kind: .enumMember),
+        TokenSpec(marker: "6️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "7️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "8️⃣", length: 5, kind: .enum),
+        TokenSpec(marker: "9️⃣", length: 6, kind: .struct, modifiers: .defaultLibrary),
+        TokenSpec(marker: "🔟", length: 4, kind: .enumMember),
+      ]
+    )
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣enum 2️⃣Maybe<3️⃣T> {
+          4️⃣case 5️⃣some(6️⃣T)
+        }
+
+        7️⃣let 8️⃣y: 9️⃣Maybe = .🔟some(0️⃣42)
+        """,
+      expected: [
+        // enum Maybe<T>
+        TokenSpec(marker: "1️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 5, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 1, kind: .identifier),
+        // case some
+        TokenSpec(marker: "4️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "5️⃣", length: 4, kind: .identifier),
+        TokenSpec(marker: "6️⃣", length: 1, kind: .typeParameter),
         // let y: Maybe = .some(42)
-        Token(line: 6, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 6, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 6, utf16index: 7, length: 5, kind: .enum),
-        Token(line: 6, utf16index: 16, length: 4, kind: .enumMember),
-        Token(line: 6, utf16index: 21, length: 2, kind: .number),
+        TokenSpec(marker: "7️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "8️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "9️⃣", length: 5, kind: .enum),
+        TokenSpec(marker: "🔟", length: 4, kind: .enumMember),
+        TokenSpec(marker: "0️⃣", length: 2, kind: .number),
       ]
     )
   }
 
   func testRegexSemanticTokens() async throws {
-    let text = """
-      let r = /a[bc]*/
-      """
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 0, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 0, utf16index: 8, length: 8, kind: .regexp),
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣let 2️⃣r = 3️⃣/a[bc]*/
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 8, kind: .regexp),
       ]
     )
   }
 
   func testOperatorDeclaration() async throws {
-    let text = """
-      infix operator ?= :ComparisonPrecedence
-      """
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 5, kind: .keyword),
-        Token(line: 0, utf16index: 6, length: 8, kind: .keyword),
-        Token(line: 0, utf16index: 15, length: 2, kind: .operator),
-        Token(line: 0, utf16index: 19, length: 20, kind: .identifier),
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣infix 2️⃣operator 3️⃣?= :4️⃣ComparisonPrecedence
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 5, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 8, kind: .keyword),
+        TokenSpec(marker: "3️⃣", length: 2, kind: .operator),
+        TokenSpec(marker: "4️⃣", length: 20, kind: .identifier),
       ]
     )
   }
 
   func testEmptyEdit() async throws {
-    let text = """
-      let x: String = "test"
-      var y = 123
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    let testClient = try await TestSourceKitLSPClient()
+    let uri = DocumentURI(for: .swift)
+    let positions = testClient.openDocument(
       """
-    openDocument(text: text)
+      1️⃣l0️⃣et 2️⃣x: 3️⃣String = 4️⃣"test"
+      5️⃣var 6️⃣y = 7️⃣123
+      """,
+      uri: uri
+    )
 
-    let before = try await performSemanticTokensRequest()
+    let expectedTokens = [
+      TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+      TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+      TokenSpec(marker: "3️⃣", length: 6, kind: .struct, modifiers: .defaultLibrary),
+      TokenSpec(marker: "4️⃣", length: 6, kind: .string),
+      TokenSpec(marker: "5️⃣", length: 3, kind: .keyword),
+      TokenSpec(marker: "6️⃣", length: 1, kind: .identifier),
+      TokenSpec(marker: "7️⃣", length: 3, kind: .number),
+    ]
 
-    let pos = Position(line: 0, utf16index: 1)
-    editDocument(range: pos..<pos, text: "", expectRefresh: false)
+    try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
 
-    let after = try await performSemanticTokensRequest()
-    XCTAssertEqual(before.tokens, after.tokens)
+    testClient.send(
+      DidChangeTextDocumentNotification(
+        textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
+        contentChanges: [TextDocumentContentChangeEvent(range: Range(positions["0️⃣"]), text: "")]
+      )
+    )
+
+    try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
   }
 
   func testReplaceUntilMiddleOfToken() async throws {
-    let text = """
-      var test = 4567
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    let testClient = try await TestSourceKitLSPClient()
+    let uri = DocumentURI(for: .swift)
+    let positions = testClient.openDocument(
       """
-    openDocument(text: text)
+      1️⃣var 2️⃣test = 3️⃣454️⃣67
+      """,
+      uri: uri
+    )
 
-    let before = try await performSemanticTokensRequest()
-    let expectedLeading = [
-      Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-      Token(line: 0, utf16index: 4, length: 4, kind: .identifier),
-    ]
-    XCTAssertEqual(
-      before.tokens,
-      expectedLeading + [
-        Token(line: 0, utf16index: 11, length: 4, kind: .number)
+    try await assertSemanticTokens(
+      uri: uri,
+      in: testClient,
+      positions: positions,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 4, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 4, kind: .number),
       ]
     )
 
-    let start = Position(line: 0, utf16index: 10)
-    let end = Position(line: 0, utf16index: 13)
-    editDocument(range: start..<end, text: " 1")
+    testClient.send(
+      DidChangeTextDocumentNotification(
+        textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
+        contentChanges: [TextDocumentContentChangeEvent(range: positions["3️⃣"]..<positions["4️⃣"], text: " 1")]
+      )
+    )
 
-    let after = try await performSemanticTokensRequest()
-    XCTAssertEqual(
-      after.tokens,
-      expectedLeading + [
-        Token(line: 0, utf16index: 11, length: 3, kind: .number)
+    let positionsAfterEdits = DocumentPositions(
+      markedText: """
+        1️⃣var 2️⃣test =  3️⃣167
+        """
+    )
+
+    try await assertSemanticTokens(
+      uri: uri,
+      in: testClient,
+      positions: positionsAfterEdits,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 4, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 3, kind: .number),
       ]
     )
   }
 
   func testReplaceUntilEndOfToken() async throws {
-    let text = """
-      fatalError("xyz")
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    let testClient = try await TestSourceKitLSPClient()
+    let uri = DocumentURI(for: .swift)
+    let positions = testClient.openDocument(
       """
-    openDocument(text: text)
+      1️⃣fatalError2️⃣(3️⃣"xyz"4️⃣)
+      """,
+      uri: uri
+    )
 
-    let before = try await performSemanticTokensRequest()
-    XCTAssertEqual(
-      before.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 10, kind: .function, modifiers: .defaultLibrary),
-        Token(line: 0, utf16index: 11, length: 5, kind: .string),
+    try await assertSemanticTokens(
+      uri: uri,
+      in: testClient,
+      positions: positions,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 10, kind: .function, modifiers: .defaultLibrary),
+        TokenSpec(marker: "3️⃣", length: 5, kind: .string),
       ]
     )
 
-    let start = Position(line: 0, utf16index: 10)
-    let end = Position(line: 0, utf16index: 16)
-    editDocument(range: start..<end, text: "(\"test\"")
+    testClient.send(
+      DidChangeTextDocumentNotification(
+        textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
+        contentChanges: [
+          TextDocumentContentChangeEvent(
+            range: positions["2️⃣"]..<positions["4️⃣"],
+            text: """
+              ("test"
+              """
+          )
+        ]
+      )
+    )
 
-    let after = try await performSemanticTokensRequest()
-    XCTAssertEqual(
-      after.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 10, kind: .function, modifiers: .defaultLibrary),
-        Token(line: 0, utf16index: 11, length: 6, kind: .string),
+    let positionsAfterEdits = DocumentPositions(
+      markedText: """
+        1️⃣fatalError2️⃣(3️⃣"test"4️⃣)
+        """
+    )
+
+    try await assertSemanticTokens(
+      uri: uri,
+      in: testClient,
+      positions: positionsAfterEdits,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 10, kind: .function, modifiers: .defaultLibrary),
+        TokenSpec(marker: "3️⃣", length: 6, kind: .string),
       ]
     )
   }
 
   func testInsertSpaceBeforeToken() async throws {
-    let text = """
-      let x: String = "test"
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    let testClient = try await TestSourceKitLSPClient()
+    let uri = DocumentURI(for: .swift)
+    let positions = testClient.openDocument(
       """
-    openDocument(text: text)
+      1️⃣let 2️⃣x: 3️⃣String = 4️⃣"test"
+      """,
+      uri: uri
+    )
 
-    let expectedBefore = [
-      SyntaxHighlightingToken(line: 0, utf16index: 0, length: 3, kind: .keyword),
-      SyntaxHighlightingToken(line: 0, utf16index: 4, length: 1, kind: .identifier),
-      SyntaxHighlightingToken(line: 0, utf16index: 7, length: 6, kind: .struct, modifiers: [.defaultLibrary]),
-      SyntaxHighlightingToken(line: 0, utf16index: 16, length: 6, kind: .string),
+    let expectedTokens = [
+      TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+      TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+      TokenSpec(marker: "3️⃣", length: 6, kind: .struct, modifiers: .defaultLibrary),
+      TokenSpec(marker: "4️⃣", length: 6, kind: .string),
     ]
-    let before = try await performSemanticTokensRequest()
-    XCTAssertEqual(before.tokens, expectedBefore)
-
-    let pos = Position(line: 0, utf16index: 0)
-    let editText = " "
-    editDocument(range: pos..<pos, text: editText, expectRefresh: false)
-
-    let after = try await performSemanticTokensRequest()
-    let expectedAfter = [
-      SyntaxHighlightingToken(line: 0, utf16index: 1, length: 3, kind: .keyword),
-      SyntaxHighlightingToken(line: 0, utf16index: 5, length: 1, kind: .identifier),
-      SyntaxHighlightingToken(line: 0, utf16index: 8, length: 6, kind: .struct, modifiers: [.defaultLibrary]),
-      SyntaxHighlightingToken(line: 0, utf16index: 17, length: 6, kind: .string),
-    ]
-    XCTAssertEqual(after.tokens, expectedAfter)
+
+    try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
+
+    testClient.send(
+      DidChangeTextDocumentNotification(
+        textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
+        contentChanges: [TextDocumentContentChangeEvent(range: Range(positions["1️⃣"]), text: " ")]
+      )
+    )
+
+    let positionsAfterEdits = DocumentPositions(
+      markedText: """
+         1️⃣let 2️⃣x: 3️⃣String = 4️⃣"test"
+        """
+    )
+
+    try await assertSemanticTokens(uri: uri, in: testClient, positions: positionsAfterEdits, expected: expectedTokens)
   }
 
   func testInsertSpaceAfterToken() async throws {
-    let text = """
-      var x = 0
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    let testClient = try await TestSourceKitLSPClient()
+    let uri = DocumentURI(for: .swift)
+    let positions = testClient.openDocument(
       """
-    openDocument(text: text)
+      1️⃣var 2️⃣x = 3️⃣04️⃣
+      """,
+      uri: uri
+    )
 
-    let before = try await performSemanticTokensRequest()
+    let expectedTokens = [
+      TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+      TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+      TokenSpec(marker: "3️⃣", length: 1, kind: .number),
+    ]
+
+    try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
 
-    let pos = Position(line: 0, utf16index: 9)
-    let editText = " "
-    editDocument(range: pos..<pos, text: editText, expectRefresh: false)
+    testClient.send(
+      DidChangeTextDocumentNotification(
+        textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
+        contentChanges: [TextDocumentContentChangeEvent(range: Range(positions["4️⃣"]), text: " ")]
+      )
+    )
 
-    let after = try await performSemanticTokensRequest()
-    XCTAssertEqual(before.tokens, after.tokens)
+    try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
   }
 
   func testInsertNewline() async throws {
-    let text = """
-      fatalError("123")
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    let testClient = try await TestSourceKitLSPClient()
+    let uri = DocumentURI(for: .swift)
+    let positions = testClient.openDocument(
       """
-    openDocument(text: text)
+      1️⃣fatalError(2️⃣"123")
+      """,
+      uri: uri
+    )
 
-    let expectedBefore = [
-      SyntaxHighlightingToken(line: 0, utf16index: 0, length: 10, kind: .function, modifiers: [.defaultLibrary]),
-      SyntaxHighlightingToken(line: 0, utf16index: 11, length: 5, kind: .string),
+    let expectedTokens = [
+      TokenSpec(marker: "1️⃣", length: 10, kind: .function, modifiers: .defaultLibrary),
+      TokenSpec(marker: "2️⃣", length: 5, kind: .string),
     ]
-    let before = try await performSemanticTokensRequest()
-    XCTAssertEqual(before.tokens, expectedBefore)
 
-    let pos = Position(line: 0, utf16index: 0)
-    editDocument(range: pos..<pos, text: "\n", expectRefresh: false)
+    try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
 
-    let after = try await performSemanticTokensRequest()
-    let expectedAfter = [
-      SyntaxHighlightingToken(line: 1, utf16index: 0, length: 10, kind: .function, modifiers: [.defaultLibrary]),
-      SyntaxHighlightingToken(line: 1, utf16index: 11, length: 5, kind: .string),
-    ]
-    XCTAssertEqual(after.tokens, expectedAfter)
+    testClient.send(
+      DidChangeTextDocumentNotification(
+        textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
+        contentChanges: [TextDocumentContentChangeEvent(range: Range(positions["1️⃣"]), text: "\n")]
+      )
+    )
+
+    let positionsAfterEdit = DocumentPositions(
+      markedText: """
+
+        1️⃣fatalError(2️⃣"123")
+        """
+    )
+
+    try await assertSemanticTokens(uri: uri, in: testClient, positions: positionsAfterEdit, expected: expectedTokens)
   }
 
   func testRemoveNewline() async throws {
-    let text = """
-      let x =
-              "abc"
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    let testClient = try await TestSourceKitLSPClient()
+    let uri = DocumentURI(for: .swift)
+    let positions = testClient.openDocument(
       """
-    openDocument(text: text)
+      1️⃣let 2️⃣x =3️⃣
+              4️⃣"abc"
+      """,
+      uri: uri
+    )
 
-    let before = try await performSemanticTokensRequest()
-    let expectedBefore = [
-      Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-      Token(line: 0, utf16index: 4, length: 1, kind: .identifier),
-      Token(line: 1, utf16index: 8, length: 5, kind: .string),
+    let expectedTokens = [
+      TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+      TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+      TokenSpec(marker: "4️⃣", length: 5, kind: .string),
     ]
-    XCTAssertEqual(before.tokens, expectedBefore)
 
-    let start = Position(line: 0, utf16index: 7)
-    let end = Position(line: 1, utf16index: 7)
-    editDocument(range: start..<end, text: "", expectRefresh: false)
+    try await assertSemanticTokens(uri: uri, in: testClient, positions: positions, expected: expectedTokens)
 
-    let after = try await performSemanticTokensRequest()
-    let expectedAfter = [
-      Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-      Token(line: 0, utf16index: 4, length: 1, kind: .identifier),
-      Token(line: 0, utf16index: 8, length: 5, kind: .string),
-    ]
-    XCTAssertEqual(after.tokens, expectedAfter)
+    testClient.send(
+      DidChangeTextDocumentNotification(
+        textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
+        contentChanges: [TextDocumentContentChangeEvent(range: positions["3️⃣"]..<positions["4️⃣"], text: " ")]
+      )
+    )
+
+    let positionsAfterEdit = DocumentPositions(
+      markedText: """
+        1️⃣let 2️⃣x = 4️⃣"abc"
+        """
+    )
+
+    try await assertSemanticTokens(uri: uri, in: testClient, positions: positionsAfterEdit, expected: expectedTokens)
   }
 
   func testInsertTokens() async throws {
-    let text = """
-      let x =
-              "abc"
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    let testClient = try await TestSourceKitLSPClient()
+    let uri = DocumentURI(for: .swift)
+    let positions = testClient.openDocument(
       """
-    openDocument(text: text)
+      1️⃣let 2️⃣x =3️⃣
+              4️⃣"abc"
+      """,
+      uri: uri
+    )
 
-    let before = try await performSemanticTokensRequest()
-    let expectedBefore = [
-      Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-      Token(line: 0, utf16index: 4, length: 1, kind: .identifier),
-      Token(line: 1, utf16index: 8, length: 5, kind: .string),
-    ]
-    XCTAssertEqual(before.tokens, expectedBefore)
-
-    let start = Position(line: 0, utf16index: 7)
-    let end = Position(line: 1, utf16index: 7)
-    editDocument(range: start..<end, text: " \"test\" +", expectRefresh: true)
-
-    let after = try await performSemanticTokensRequest()
-    let expectedAfter: [Token] = [
-      Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-      Token(line: 0, utf16index: 4, length: 1, kind: .identifier),
-      Token(line: 0, utf16index: 8, length: 6, kind: .string),
-      Token(line: 0, utf16index: 15, length: 1, kind: .method, modifiers: [.defaultLibrary, .static]),
-      Token(line: 0, utf16index: 17, length: 5, kind: .string),
-    ]
-    XCTAssertEqual(after.tokens, expectedAfter)
+    try await assertSemanticTokens(
+      uri: uri,
+      in: testClient,
+      positions: positions,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "4️⃣", length: 5, kind: .string),
+      ]
+    )
+
+    testClient.send(
+      DidChangeTextDocumentNotification(
+        textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
+        contentChanges: [TextDocumentContentChangeEvent(range: positions["3️⃣"]..<positions["4️⃣"], text: #" "test" + "#)]
+      )
+    )
+
+    let positionsAfterEdits = DocumentPositions(
+      markedText: """
+        1️⃣let 2️⃣x = 3️⃣"test" 4️⃣+ 5️⃣"abc"
+        """
+    )
+
+    try await assertSemanticTokens(
+      uri: uri,
+      in: testClient,
+      positions: positionsAfterEdits,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 6, kind: .string),
+        TokenSpec(marker: "4️⃣", length: 1, kind: .method, modifiers: [.defaultLibrary, .static]),
+        TokenSpec(marker: "5️⃣", length: 5, kind: .string),
+      ]
+    )
   }
 
   func testSemanticMultiEdit() async throws {
-    let text = """
-      let x = "abc"
-      let y = x
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    let testClient = try await TestSourceKitLSPClient()
+    let uri = DocumentURI(for: .swift)
+    let positions = testClient.openDocument(
       """
-    openDocument(text: text)
+      1️⃣let 2️⃣x3️⃣ = 4️⃣"abc"
+      5️⃣let 6️⃣y = 7️⃣x8️⃣
+      """,
+      uri: uri
+    )
 
-    let before = try await performSemanticTokensRequest()
-    XCTAssertEqual(
-      before.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 0, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 0, utf16index: 8, length: 5, kind: .string),
-        Token(line: 1, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 1, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 1, utf16index: 8, length: 1, kind: .variable),
+    try await assertSemanticTokens(
+      uri: uri,
+      in: testClient,
+      positions: positions,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "4️⃣", length: 5, kind: .string),
+        TokenSpec(marker: "5️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "6️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "7️⃣", length: 1, kind: .variable),
       ]
     )
 
     let newName = "renamed"
-    editDocument(
-      changes: [
-        TextDocumentContentChangeEvent(
-          range: Position(line: 0, utf16index: 4)..<Position(line: 0, utf16index: 5),
-          text: newName
-        ),
-        TextDocumentContentChangeEvent(
-          range: Position(line: 1, utf16index: 8)..<Position(line: 1, utf16index: 9),
-          text: newName
-        ),
-      ],
-      expectRefresh: true
-    )
-
-    let after = try await performSemanticTokensRequest()
-    XCTAssertEqual(
-      after.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 0, utf16index: 4, length: 7, kind: .identifier),
-        Token(line: 0, utf16index: 14, length: 5, kind: .string),
-        Token(line: 1, utf16index: 0, length: 3, kind: .keyword),
-        Token(line: 1, utf16index: 4, length: 1, kind: .identifier),
-        Token(line: 1, utf16index: 8, length: 7, kind: .variable),
+    testClient.send(
+      DidChangeTextDocumentNotification(
+        textDocument: VersionedTextDocumentIdentifier(uri, version: 2),
+        contentChanges: [
+          TextDocumentContentChangeEvent(range: positions["2️⃣"]..<positions["3️⃣"], text: newName),
+          TextDocumentContentChangeEvent(range: positions["7️⃣"]..<positions["8️⃣"], text: newName),
+        ]
+      )
+    )
+
+    let positionsAfterEdits = DocumentPositions(
+      markedText: """
+        1️⃣let 2️⃣renamed = 4️⃣"abc"
+        5️⃣let 6️⃣y = 7️⃣renamed
+        """
+    )
+
+    try await assertSemanticTokens(
+      uri: uri,
+      in: testClient,
+      positions: positionsAfterEdits,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 7, kind: .identifier),
+        TokenSpec(marker: "4️⃣", length: 5, kind: .string),
+        TokenSpec(marker: "5️⃣", length: 3, kind: .keyword),
+        TokenSpec(marker: "6️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "7️⃣", length: 7, kind: .variable),
       ]
     )
   }
 
   func testActor() async throws {
-    let text = """
-      actor MyActor {}
-
-      struct MyStruct {}
-
-      func t(
-          x: MyActor,
-          y: MyStruct
-      ) {}
-      """
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
 
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 5, kind: .keyword),
-        Token(line: 0, utf16index: 6, length: 7, kind: .identifier),
-        Token(line: 2, utf16index: 0, length: 6, kind: .keyword),
-        Token(line: 2, utf16index: 7, length: 8, kind: .identifier),
-        Token(line: 4, utf16index: 0, length: 4, kind: .keyword),
-        Token(line: 4, utf16index: 5, length: 1, kind: .identifier),
-        Token(line: 5, utf16index: 4, length: 1, kind: .function),
-        Token(line: 5, utf16index: 7, length: 7, kind: .actor),
-        Token(line: 6, utf16index: 4, length: 1, kind: .function),
-        Token(line: 6, utf16index: 7, length: 8, kind: .struct),
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣actor 2️⃣MyActor {}
+
+        3️⃣func 4️⃣t(5️⃣x: 6️⃣MyActor) {}
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 5, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 7, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "4️⃣", length: 1, kind: .identifier),
+        TokenSpec(marker: "5️⃣", length: 1, kind: .function),
+        TokenSpec(marker: "6️⃣", length: 7, kind: .actor),
       ]
     )
   }
 
   func testArgumentLabels() async throws {
-    let text = """
-      func foo(arg: Int) {}
-      foo(arg: 1)
-      """
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
 
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 4, kind: .keyword),
-        Token(line: 0, utf16index: 5, length: 3, kind: .identifier),
-        Token(line: 0, utf16index: 9, length: 3, kind: .function),
-        Token(line: 0, utf16index: 14, length: 3, kind: .struct, modifiers: .defaultLibrary),
-        Token(line: 1, utf16index: 0, length: 3, kind: .function),
-        Token(line: 1, utf16index: 4, length: 3, kind: .function),
-        Token(line: 1, utf16index: 9, length: 1, kind: .number),
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣func 2️⃣foo(3️⃣arg: 4️⃣Int) {}
+        5️⃣foo(6️⃣arg: 7️⃣1)
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 3, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 3, kind: .function),
+        TokenSpec(marker: "4️⃣", length: 3, kind: .struct, modifiers: .defaultLibrary),
+        TokenSpec(marker: "5️⃣", length: 3, kind: .function),
+        TokenSpec(marker: "6️⃣", length: 3, kind: .function),
+        TokenSpec(marker: "7️⃣", length: 1, kind: .number),
       ]
     )
   }
 
   func testFunctionDeclarationWithFirstAndSecondName() async throws {
-    let text = """
-      func foo(arg internalName: Int) {}
-      """
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
 
-    let tokens = try await openAndPerformSemanticTokensRequest(text: text)
-    XCTAssertEqual(
-      tokens.tokens,
-      [
-        Token(line: 0, utf16index: 0, length: 4, kind: .keyword),
-        Token(line: 0, utf16index: 5, length: 3, kind: .identifier),
-        Token(line: 0, utf16index: 9, length: 3, kind: .function),
-        Token(line: 0, utf16index: 13, length: 12, kind: .identifier),
-        Token(line: 0, utf16index: 27, length: 3, kind: .struct, modifiers: .defaultLibrary),
+    try await assertSemanticTokens(
+      markedContents: """
+        1️⃣func 2️⃣foo(3️⃣arg 4️⃣internalName: 5️⃣Int) {}
+        """,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 4, kind: .keyword),
+        TokenSpec(marker: "2️⃣", length: 3, kind: .identifier),
+        TokenSpec(marker: "3️⃣", length: 3, kind: .function),
+        TokenSpec(marker: "4️⃣", length: 12, kind: .identifier),
+        TokenSpec(marker: "5️⃣", length: 3, kind: .struct, modifiers: .defaultLibrary),
+      ]
+    )
+  }
+
+  func testClang() async throws {
+    try await SkipUnless.sourcekitdHasSemanticTokensRequest()
+
+    try await assertSemanticTokens(
+      markedContents: """
+        int 1️⃣main() {}
+        """,
+      language: .c,
+      expected: [
+        TokenSpec(marker: "1️⃣", length: 4, kind: .function, modifiers: [.declaration, .definition, .globalScope])
       ]
     )
   }
 }
 
-extension Token {
-  fileprivate init(
-    line: Int,
-    utf16index: Int,
+fileprivate struct TokenSpec {
+  let marker: String
+  let length: Int
+  let kind: SemanticTokenTypes
+  let modifiers: SemanticTokenModifiers
+
+  init(
+    marker: String,
     length: Int,
     kind: SemanticTokenTypes,
     modifiers: SemanticTokenModifiers = []
   ) {
-    self.init(
-      start: Position(line: line, utf16index: utf16index),
-      utf16length: length,
-      kind: kind,
-      modifiers: modifiers
+    self.marker = marker
+    self.length = length
+    self.kind = kind
+    self.modifiers = modifiers
+  }
+}
+
+fileprivate func assertSemanticTokens(
+  markedContents: String,
+  language: Language = .swift,
+  range: (startMarker: String, endMarker: String)? = nil,
+  expected: [TokenSpec],
+  file: StaticString = #filePath,
+  line: UInt = #line
+) async throws {
+  let testClient = try await TestSourceKitLSPClient()
+  let uri = DocumentURI(for: language)
+  let positions = testClient.openDocument(markedContents, uri: uri)
+
+  let response: DocumentSemanticTokensResponse?
+  if let range {
+    response = try await testClient.send(
+      DocumentSemanticTokensRangeRequest(
+        textDocument: TextDocumentIdentifier(uri),
+        range: positions[range.startMarker]..<positions[range.endMarker]
+      )
     )
+  } else {
+    response = try await testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri)))
+  }
+
+  let expectedTokens = expected.map {
+    Token(start: positions[$0.marker], utf16length: $0.length, kind: $0.kind, modifiers: $0.modifiers)
+  }
+  XCTAssertEqual(
+    SyntaxHighlightingTokens(lspEncodedTokens: try unwrap(response, file: file, line: line).data).tokens,
+    expectedTokens,
+    file: file,
+    line: line
+  )
+}
+
+fileprivate func assertSemanticTokens(
+  uri: DocumentURI,
+  in testClient: TestSourceKitLSPClient,
+  positions: DocumentPositions,
+  expected: [TokenSpec],
+  file: StaticString = #filePath,
+  line: UInt = #line
+) async throws {
+  let response = try await unwrap(
+    testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri))),
+    file: file,
+    line: line
+  )
+  let expectedTokens = expected.map {
+    Token(start: positions[$0.marker], utf16length: $0.length, kind: $0.kind, modifiers: $0.modifiers)
   }
+  XCTAssertEqual(
+    SyntaxHighlightingTokens(lspEncodedTokens: response.data).tokens,
+    expectedTokens,
+    file: file,
+    line: line
+  )
 }