[Dependency Scanning] Fix optional import statement serialization logic

include/swift/DependencyScan/DependencyScanImpl.h

@@ -65,6 +65,9 @@ struct swiftscan_dependency_info_s {
   /// The list of source import infos.
   swiftscan_import_info_set_t *imports;
 
+  /// The list of source optional import infos.
+  swiftscan_import_info_set_t *optional_imports;
+
   /// Specific details of a particular kind of module.
   swiftscan_module_details_t details;
 };

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -41,7 +41,7 @@ using llvm::BCVBR;
 const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
 const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 10;
 /// Increment this on every change.
-const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 3;
+const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 4;
 
 /// Various identifiers in this format will rely on having their strings mapped
 /// using this ID.

lib/DependencyScan/DependencyScanJSON.cpp

@@ -228,9 +228,13 @@ void writeLinkLibraries(llvm::raw_ostream &out,
 
 void writeImportInfos(llvm::raw_ostream &out,
                       const swiftscan_import_info_set_t *imports,
-                      unsigned indentLevel, bool trailingComma) {
+                      bool optional, unsigned indentLevel,
+                      bool trailingComma) {
   out.indent(indentLevel * 2);
-  out << "\"imports\": ";
+  if (optional)
+    out << "\"optionalImports\": ";
+  else
+    out << "\"imports\": ";
   out << "[\n";
 
   for (size_t i = 0; i < imports->count; ++i) {
@@ -441,7 +445,9 @@ void writeJSON(llvm::raw_ostream &out,
                         /*trailingComma=*/true);
       writeLinkLibraries(out, moduleInfo.link_libraries,
                          3, /*trailingComma=*/true);
-      writeImportInfos(out, moduleInfo.imports,
+      writeImportInfos(out, moduleInfo.imports, /*optional*/ false,
+                       3, /*trailingComma=*/true);
+      writeImportInfos(out, moduleInfo.optional_imports, /*optional*/ true,
                        3, /*trailingComma=*/true);
     }
     // Swift and Clang-specific details.

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -49,8 +49,8 @@ class ModuleDependenciesCacheDeserializer {
   std::vector<std::vector<uint64_t>> ArraysOfMacroDependenciesIDs;
   std::vector<ScannerImportStatementInfo> ImportStatements;
   std::vector<std::vector<uint64_t>> ArraysOfImportStatementIDs;
-  std::vector<std::vector<uint64_t>> ArraysOfSearchPathIDs;
   std::vector<std::vector<uint64_t>> ArraysOfOptionalImportStatementIDs;
+  std::vector<std::vector<uint64_t>> ArraysOfSearchPathIDs;
 
   llvm::BitstreamCursor Cursor;
   SmallVector<uint64_t, 64> Scratch;
@@ -1170,6 +1170,7 @@ class ModuleDependenciesCacheSerializer {
   unsigned writeImportStatementInfos(const ModuleDependencyInfo &dependencyInfo,
                                      bool optional);
   void writeImportStatementInfosArray(unsigned startIndex, unsigned count);
+  void writeOptionalImportStatementInfosArray(unsigned startIndex, unsigned count);
 
   void writeModuleInfo(ModuleDependencyID moduleID,
                        const ModuleDependencyInfo &dependencyInfo);
@@ -1478,7 +1479,7 @@ void ModuleDependenciesCacheSerializer::writeImportStatementInfos(
       }
       auto optionalEntries = optionalImportInfoArrayMap.at(moduleID);
       if (optionalEntries.second != 0) {
-        writeImportStatementInfosArray(optionalEntries.first, optionalEntries.second);
+        writeOptionalImportStatementInfosArray(optionalEntries.first, optionalEntries.second);
         OptionalImportInfosArrayIDsMap.insert({moduleID, lastOptionalImportInfoArrayIndex++});
       }
     }
@@ -1530,6 +1531,15 @@ void ModuleDependenciesCacheSerializer::writeImportStatementInfosArray(
       Out, ScratchRecord, AbbrCodes[ImportStatementArrayLayout::Code], vec);
 }
 
+void ModuleDependenciesCacheSerializer::writeOptionalImportStatementInfosArray(
+    unsigned startIndex, unsigned count) {
+  using namespace graph_block;
+  std::vector<unsigned> vec(count);
+  std::iota(vec.begin(), vec.end(), startIndex);
+  OptionalImportStatementArrayLayout::emitRecord(
+      Out, ScratchRecord, AbbrCodes[OptionalImportStatementArrayLayout::Code], vec);
+}
+
 void ModuleDependenciesCacheSerializer::writeModuleInfo(
     ModuleDependencyID moduleID, const ModuleDependencyInfo &dependencyInfo) {
   using namespace graph_block;
@@ -1965,6 +1975,7 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
   registerRecordAbbr<SearchPathArrayLayout>();
   registerRecordAbbr<ImportStatementLayout>();
   registerRecordAbbr<ImportStatementArrayLayout>();
+  registerRecordAbbr<OptionalImportStatementArrayLayout>();
   registerRecordAbbr<ModuleInfoLayout>();
   registerRecordAbbr<SwiftSourceModuleDetailsLayout>();
   registerRecordAbbr<SwiftInterfaceModuleDetailsLayout>();

lib/DependencyScan/ScanDependencies.cpp

@@ -922,37 +922,44 @@ static swiftscan_dependency_graph_t generateFullDependencyGraph(
     }
     moduleInfo->link_libraries = linkLibrarySet;
 
-    // Create source import infos set for this module
-    auto imports = moduleDependencyInfo.getModuleImports();
-    swiftscan_import_info_set_t *importInfoSet =
-        new swiftscan_import_info_set_t;
-    importInfoSet->count = imports.size();
-    importInfoSet->imports = new swiftscan_import_info_t[importInfoSet->count];
-    for (size_t i = 0; i < imports.size(); ++i) {
-      const auto &ii = imports[i];
-      swiftscan_import_info_s *iInfo = new swiftscan_import_info_s;
-      iInfo->import_identifier = create_clone(ii.importIdentifier.c_str());
-      iInfo->access_level =
-          static_cast<swiftscan_access_level_t>(ii.accessLevel);
-
-      const auto &sourceLocations = ii.importLocations;
-      swiftscan_source_location_set_t *sourceLocSet =
-          new swiftscan_source_location_set_t;
-      sourceLocSet->count = sourceLocations.size();
-      sourceLocSet->source_locations =
-          new swiftscan_source_location_t[sourceLocSet->count];
-      for (size_t j = 0; j < sourceLocations.size(); ++j) {
-        const auto &sl = sourceLocations[j];
-        swiftscan_source_location_s *slInfo = new swiftscan_source_location_s;
-        slInfo->buffer_identifier = create_clone(sl.bufferIdentifier.c_str());
-        slInfo->line_number = sl.lineNumber;
-        slInfo->column_number = sl.columnNumber;
-        sourceLocSet->source_locations[j] = slInfo;
+    auto createImportSetInfo = [&](ArrayRef<ScannerImportStatementInfo> imports)
+        -> swiftscan_import_info_set_t * {
+      swiftscan_import_info_set_t *importInfoSet =
+          new swiftscan_import_info_set_t;
+      importInfoSet->count = imports.size();
+      importInfoSet->imports =
+          new swiftscan_import_info_t[importInfoSet->count];
+      for (size_t i = 0; i < imports.size(); ++i) {
+        const auto &ii = imports[i];
+        swiftscan_import_info_s *iInfo = new swiftscan_import_info_s;
+        iInfo->import_identifier = create_clone(ii.importIdentifier.c_str());
+        iInfo->access_level =
+            static_cast<swiftscan_access_level_t>(ii.accessLevel);
+
+        const auto &sourceLocations = ii.importLocations;
+        swiftscan_source_location_set_t *sourceLocSet =
+            new swiftscan_source_location_set_t;
+        sourceLocSet->count = sourceLocations.size();
+        sourceLocSet->source_locations =
+            new swiftscan_source_location_t[sourceLocSet->count];
+        for (size_t j = 0; j < sourceLocations.size(); ++j) {
+          const auto &sl = sourceLocations[j];
+          swiftscan_source_location_s *slInfo = new swiftscan_source_location_s;
+          slInfo->buffer_identifier = create_clone(sl.bufferIdentifier.c_str());
+          slInfo->line_number = sl.lineNumber;
+          slInfo->column_number = sl.columnNumber;
+          sourceLocSet->source_locations[j] = slInfo;
+        }
+        iInfo->source_locations = sourceLocSet;
+        importInfoSet->imports[i] = iInfo;
       }
-      iInfo->source_locations = sourceLocSet;
-      importInfoSet->imports[i] = iInfo;
-    }
-    moduleInfo->imports = importInfoSet;
+      return importInfoSet;
+    };
+    // Create source import infos set for this module
+    moduleInfo->imports =
+        createImportSetInfo(moduleDependencyInfo.getModuleImports());
+    moduleInfo->optional_imports =
+        createImportSetInfo(moduleDependencyInfo.getOptionalModuleImports());
   }
 
   swiftscan_dependency_graph_t result = new swiftscan_dependency_graph_s;

test/ScanDependencies/serialized_imports.swift

@@ -3,12 +3,14 @@
 // RUN: %empty-directory(%t/module-cache)
 
 // Run the scanner once, emitting the serialized scanner cache
-// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -Rdependency-scan-cache -serialize-dependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -enable-cross-import-overlays 2>&1 | %FileCheck %s -check-prefix CHECK-REMARK-SAVE
+// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -Rdependency-scan-cache -serialize-dependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -enable-cross-import-overlays -module-name deps 2>&1 | %FileCheck %s -check-prefix CHECK-REMARK-SAVE
 // RUN: llvm-bcanalyzer --dump %t/cache.moddepcache > %t/cache.moddepcache.initial.dump.txt
+// RUN: %validate-json %t/deps.json | %FileCheck %s -check-prefix CHECK-IMPORTS
 
 // Run the scanner again, but now re-using previously-serialized cache
-// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -Rdependency-scan-cache -serialize-dependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -enable-cross-import-overlays 2>&1 | %FileCheck %s -check-prefix CHECK-REMARK-LOAD
+// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -Rdependency-scan-cache -serialize-dependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/module-cache %s -o %t/deps_incremental.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -enable-cross-import-overlays -module-name deps 2>&1 | %FileCheck %s -check-prefix CHECK-REMARK-LOAD
 // RUN: llvm-bcanalyzer --dump %t/cache.moddepcache > %t/cache.moddepcache.dump.txt
+// RUN: %validate-json %t/deps_incremental.json | %FileCheck %s -check-prefix CHECK-IMPORTS
 
 // Ensure that the initial scan, and the secondary scan which just re-used the initial scan's results report
 // the same number of import statement nodes, ensuring that serialization-deserialization did not affect
@@ -22,8 +24,32 @@
 // CHECK-REMARK-SAVE: remark: Incremental module scan: Serializing module scanning dependency cache to:
 // CHECK-REMARK-LOAD: remark: Incremental module scan: Re-using serialized module scanning dependency cache from:
 
-import E
-
-
-
+// CHECK-IMPORTS:      "modulePath": "deps.swiftmodule",
+// CHECK-IMPORTS:      "imports": [
+// CHECK-IMPORTS-NEXT:        {
+// CHECK-IMPORTS-NEXT:          "identifier": "Swift",
+// CHECK-IMPORTS-NEXT:          "accessLevel": "public"
+// CHECK-IMPORTS-NEXT:        },
+// CHECK-IMPORTS-NEXT:        {
+// CHECK-IMPORTS-NEXT:          "identifier": "SwiftOnoneSupport",
+// CHECK-IMPORTS-NEXT:          "accessLevel": "public"
+// CHECK-IMPORTS-NEXT:        },
+// CHECK-IMPORTS-NEXT:        {
+// CHECK-IMPORTS-NEXT:          "identifier": "E",
+// CHECK-IMPORTS-NEXT:          "accessLevel": "public",
+// CHECK-IMPORTS-NEXT:          "importLocations": [
+// CHECK-IMPORTS-NEXT:            {
+// CHECK-IMPORTS-NEXT:              "bufferIdentifier": "{{.*}}serialized_imports.swift",
+// CHECK-IMPORTS-NEXT:              "linuNumber": 54,
+// CHECK-IMPORTS-NEXT:              "columnNumber": 8
+// CHECK-IMPORTS-NEXT:            }
+
+// CHECK-IMPORTS:          "optionalImports": [
+// CHECK-IMPORTS-NEXT:        {
+// CHECK-IMPORTS-NEXT:          "identifier": "E_Private",
+// CHECK-IMPORTS-NEXT:          "accessLevel": "public"
+// CHECK-IMPORTS-NEXT:        }
+// CHECK-IMPORTS-NEXT:      ],
 
+import E
+import E.Private