Merge pull request #696 from artemcm/IncrementalExplicitBuilds

artemcm · web-flow · commit 41f1bb34e775 · 2021-06-14T10:30:56.000-07:00
[Explicit Module Builds] Add libSwiftScan API for scanner cache serialization/deserialization
diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h
@@ -200,17 +200,20 @@ typedef struct {
   //=== Scanner Functions ---------------------------------------------------===//  
   swiftscan_scanner_t (*swiftscan_scanner_create)(void);
   void (*swiftscan_scanner_dispose)(swiftscan_scanner_t);
-
   swiftscan_dependency_graph_t
   (*swiftscan_dependency_graph_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
-
   swiftscan_batch_scan_result_t *
   (*swiftscan_batch_scan_result_create)(swiftscan_scanner_t,
                                         swiftscan_batch_scan_input_t *,
                                         swiftscan_scan_invocation_t);
-
   swiftscan_import_set_t
   (*swiftscan_import_set_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
+
+  //=== Scanner Cache Functions ---------------------------------------------===//
+  void (*swiftscan_scanner_cache_serialize)(swiftscan_scanner_t scanner, const char * path);
+  bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path);
+  void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner);
+
 } swiftscan_functions_t;
 
 #endif // SWIFT_C_DEPENDENCY_SCAN_H
diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift
@@ -73,6 +73,34 @@ public class InterModuleDependencyOracle {
     }
   }
 
+  @_spi(Testing) public func serializeScannerCache(to path: AbsolutePath) {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to serialize scanner cache with no scanner instance.")
+    }
+    if swiftScan.canLoadStoreScannerCache() {
+      swiftScan.serializeScannerCache(to: path)
+    }
+  }
+
+  @_spi(Testing) public func loadScannerCache(from path: AbsolutePath) -> Bool {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to load scanner cache with no scanner instance.")
+    }
+    if swiftScan.canLoadStoreScannerCache() {
+      return swiftScan.loadScannerCache(from: path)
+    }
+    return false
+  }
+
+  @_spi(Testing) public func resetScannerCache() {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to reset scanner cache with no scanner instance.")
+    }
+    if swiftScan.canLoadStoreScannerCache() {
+      swiftScan.resetScannerCache()
+    }
+  }
+
   private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil }
 
   /// Queue to sunchronize accesses to the scanner
diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift
@@ -93,7 +93,7 @@ extension Driver {
          .privateSwiftInterface, .swiftSourceInfoFile, .diagnostics, .objcHeader, .swiftDeps,
          .remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm,
          .pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
-         .indexUnitOutputPath, nil:
+         .indexUnitOutputPath, .modDepCache, nil:
       return false
     }
   }
@@ -444,7 +444,7 @@ extension FileType {
          .diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
          .yamlOptimizationRecord, .bitstreamOptimizationRecord, .swiftInterface,
          .privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts,
-         .indexUnitOutputPath:
+         .indexUnitOutputPath, .modDepCache:
       fatalError("Output type can never be a primary output")
     }
   }
diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift
@@ -181,6 +181,26 @@ internal final class SwiftScan {
     return resultGraphMap
   }
 
+  @_spi(Testing) public func canLoadStoreScannerCache() -> Bool {
+    return api.swiftscan_scanner_cache_load != nil &&
+           api.swiftscan_scanner_cache_serialize != nil &&
+           api.swiftscan_scanner_cache_reset != nil
+  }
+
+  func serializeScannerCache(to path: AbsolutePath) {
+    api.swiftscan_scanner_cache_serialize(scanner,
+                                          path.description.cString(using: String.Encoding.utf8))
+  }
+
+  func loadScannerCache(from path: AbsolutePath) -> Bool {
+    return api.swiftscan_scanner_cache_load(scanner,
+                                            path.description.cString(using: String.Encoding.utf8))
+  }
+
+  func resetScannerCache() {
+    api.swiftscan_scanner_cache_reset(scanner)
+  }
+
   @_spi(Testing) public func canQuerySupportedArguments() -> Bool {
     return api.swiftscan_compiler_supported_arguments_query != nil &&
            api.swiftscan_string_set_dispose != nil
@@ -226,13 +246,22 @@ private extension swiftscan_functions_t {
       }
       return sym
     }
+    // Supported features/flags query
     self.swiftscan_string_set_dispose =
       try loadOptional("swiftscan_string_set_dispose")
     self.swiftscan_compiler_supported_arguments_query =
       try loadOptional("swiftscan_compiler_supported_arguments_query")
     self.swiftscan_compiler_supported_features_query =
       try loadOptional("swiftscan_compiler_supported_features_query")
 
+    // Dependency scanner serialization/deserialization features
+    self.swiftscan_scanner_cache_serialize =
+      try loadOptional("swiftscan_scanner_cache_serialize")
+    self.swiftscan_scanner_cache_load =
+      try loadOptional("swiftscan_scanner_cache_load")
+    self.swiftscan_scanner_cache_reset =
+      try loadOptional("swiftscan_scanner_cache_reset")
+
     // MARK: Required Methods
     func loadRequired<T>(_ symbol: String) throws -> T {
       guard let sym: T = dlsym(swiftscan, symbol: symbol) else {
diff --git a/Sources/SwiftDriver/Utilities/FileType.swift b/Sources/SwiftDriver/Utilities/FileType.swift
@@ -81,6 +81,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
   /// Swift dependencies file.
   case swiftDeps = "swiftdeps"
 
+  /// Serialized dependency scanner state
+  case modDepCache = "moddepcache"
+
   /// Remapping file
   case remap
 
@@ -109,20 +112,20 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
   /// Swift section of the internal wiki.
   case moduleTrace = "trace.json"
 
-  /// Indexing data directory.
+  /// Indexing data directory
   ///
-  /// The extension isn't real.
+  /// The extension isn't real, rather this FileType specifies a directory path.
   case indexData
 
   /// Output path to record in the indexing data store
   ///
   /// This is only needed for use as a key in the output file map.
   case indexUnitOutputPath
 
-  /// Optimization record.
+  /// Optimization record
   case yamlOptimizationRecord = "opt.yaml"
 
-  /// Bitstream optimization record.
+  /// Bitstream optimization record
   case bitstreamOptimizationRecord = "opt.bitstream"
 
   /// Clang compiler module file
@@ -169,6 +172,9 @@ extension FileType: CustomStringConvertible {
     case .swiftDeps:
       return "swift-dependencies"
 
+    case .modDepCache:
+        return "dependency-scanner-cache"
+
     case .jsonDependencies:
       return "json-dependencies"
 
@@ -218,7 +224,7 @@ extension FileType {
          .swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord,
          .swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .jsonDependencies,
          .clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts,
-         .indexUnitOutputPath:
+         .indexUnitOutputPath, .modDepCache:
       return false
     }
   }
@@ -289,6 +295,8 @@ extension FileType {
       return "objc-header"
     case .swiftDeps:
       return "swift-dependencies"
+    case .modDepCache:
+      return "dependency-scanner-cache"
     case .jsonDependencies:
       return "json-dependencies"
     case .jsonTargetInfo:
@@ -327,7 +335,7 @@ extension FileType {
     case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule,
          .swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics,
          .pcm, .swiftDeps, .remap, .indexData, .bitstreamOptimizationRecord,
-         .indexUnitOutputPath:
+         .indexUnitOutputPath, .modDepCache:
       return false
     }
   }
@@ -341,7 +349,7 @@ extension FileType {
     case .swift, .sil, .sib, .ast, .image, .dSYM, .dependencies, .autolink,
          .swiftModule, .swiftDocumentation, .swiftInterface, .privateSwiftInterface,
          .swiftSourceInfoFile, .raw_sil, .raw_sib, .diagnostics, .objcHeader, .swiftDeps, .remap,
-         .importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord,
+         .importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord, .modDepCache,
          .bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies, .clangModuleMap,
          .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, .indexUnitOutputPath:
       return false
diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift
@@ -642,24 +642,31 @@ final class ExplicitModuleBuildTests: XCTestCase {
     }
   }
 
-  /// Test the libSwiftScan dependency scanning.
-  func testDependencyScanning() throws {
+  private func getDriverArtifactsForScanning() throws -> (stdLibPath: AbsolutePath,
+                                                          shimsPath: AbsolutePath,
+                                                          toolchain: Toolchain,
+                                                          hostTriple: Triple) {
     // Just instantiating to get at the toolchain path
     let driver = try Driver(args: ["swiftc", "-experimental-explicit-module-build",
                                    "-module-name", "testDependencyScanning",
                                    "test.swift"])
     let (stdLibPath, shimsPath) = try getStdlibShimsPaths(driver)
-
     XCTAssertTrue(localFileSystem.exists(stdLibPath),
                   "expected Swift StdLib at: \(stdLibPath.description)")
     XCTAssertTrue(localFileSystem.exists(shimsPath),
                   "expected Swift Shims at: \(shimsPath.description)")
+    return (stdLibPath, shimsPath, driver.toolchain, driver.hostTriple)
+  }
+
+  /// Test the libSwiftScan dependency scanning.
+  func testDependencyScanning() throws {
+    let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
 
     // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
     // queries.
     let dependencyOracle = InterModuleDependencyOracle()
-    let scanLibPath = try Driver.getScanLibPath(of: driver.toolchain,
-                                                hostTriple: driver.hostTriple,
+    let scanLibPath = try Driver.getScanLibPath(of: toolchain,
+                                                hostTriple: hostTriple,
                                                 env: ProcessEnv.vars)
     guard try dependencyOracle
             .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -694,8 +701,8 @@ final class ExplicitModuleBuildTests: XCTestCase {
       // Module `X` is only imported on Darwin when:
       // #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 110000
       let expectedNumberOfDependencies: Int
-      if driver.targetTriple.isMacOSX,
-         driver.targetTriple.version(for: .macOS) >= Triple.Version(11, 0, 0) {
+      if hostTriple.isMacOSX,
+         hostTriple.version(for: .macOS) >= Triple.Version(11, 0, 0) {
         expectedNumberOfDependencies = 11
       } else {
         expectedNumberOfDependencies = 12
@@ -733,6 +740,64 @@ final class ExplicitModuleBuildTests: XCTestCase {
     }
   }
 
+
+  /// Test the libSwiftScan dependency scanning.
+  func testDependencyScanReuseCache() throws {
+    let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+    try withTemporaryDirectory { path in
+      let cacheSavePath = path.appending(component: "saved.moddepcache")
+      let main = path.appending(component: "testDependencyScanning.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import C;"
+        $0 <<< "import E;"
+        $0 <<< "import G;"
+      }
+      let packageRootPath = URL(fileURLWithPath: #file).pathComponents
+        .prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst()
+      let testInputsPath = packageRootPath + "/TestInputs"
+      let cHeadersPath : String = testInputsPath + "/ExplicitModuleBuilds/CHeaders"
+      let swiftModuleInterfacesPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift"
+      let scannerCommand = ["-scan-dependencies",
+                            "-I", cHeadersPath,
+                            "-I", swiftModuleInterfacesPath,
+                            "-I", stdLibPath.description,
+                            "-I", shimsPath.description,
+                            main.pathString]
+
+      let scanLibPath = try Driver.getScanLibPath(of: toolchain,
+                                                  hostTriple: hostTriple,
+                                                  env: ProcessEnv.vars)
+      // Run the first scan and serialize the cache contents.
+      let firstDependencyOracle = InterModuleDependencyOracle()
+      guard try firstDependencyOracle
+              .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
+                                             swiftScanLibPath: scanLibPath) else {
+        XCTFail("Dependency scanner library not found")
+        return
+      }
+
+      let firstScanGraph =
+        try! firstDependencyOracle.getDependencies(workingDirectory: path,
+                                              commandLine: scannerCommand)
+      firstDependencyOracle.serializeScannerCache(to: cacheSavePath)
+
+      // Run the second scan, re-using the serialized cache contents.
+      let secondDependencyOracle = InterModuleDependencyOracle()
+      guard try secondDependencyOracle
+              .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
+                                             swiftScanLibPath: scanLibPath) else {
+        XCTFail("Dependency scanner library not found")
+        return
+      }
+      XCTAssertFalse(secondDependencyOracle.loadScannerCache(from: cacheSavePath))
+      let secondScanGraph =
+        try! secondDependencyOracle.getDependencies(workingDirectory: path,
+                                                    commandLine: scannerCommand)
+
+      XCTAssertTrue(firstScanGraph.modules.count == secondScanGraph.modules.count)
+    }
+  }
+
   func testDependencyGraphMerge() throws {
     let moduleDependencyGraph1 =
           try JSONDecoder().decode(

Original file line number	Diff line number	Diff line change
`@@ -93,7 +93,7 @@ extension Driver {`
`93`	`93`	`.privateSwiftInterface, .swiftSourceInfoFile, .diagnostics, .objcHeader, .swiftDeps,`
`94`	`94`	`.remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm,`
`95`	`95`	`.pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,`
`96`		`- .indexUnitOutputPath, nil:`
	`96`	`+ .indexUnitOutputPath, .modDepCache, nil:`
`97`	`97`	`return false`
`98`	`98`	`}`
`99`	`99`	`}`
`@@ -444,7 +444,7 @@ extension FileType {`
`444`	`444`	`.diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,`
`445`	`445`	`.yamlOptimizationRecord, .bitstreamOptimizationRecord, .swiftInterface,`
`446`	`446`	`.privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts,`
`447`		`- .indexUnitOutputPath:`
	`447`	`+ .indexUnitOutputPath, .modDepCache:`
`448`	`448`	`fatalError("Output type can never be a primary output")`
`449`	`449`	`}`
`450`	`450`	`}`