From cdeef58e0fadb951f739d84393752587bedec9f5 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 22 Jan 2024 14:43:52 -0800 Subject: [PATCH 1/4] [Caching] Create clang importer from cc1 args directly When caching build is enabled, teach dependency scanner to report command-lines with `-direct-clang-cc1-module-build` so the later compilation can instantiate clang importer with cc1 args directly. This avoids running clang driver code, which might involve file system lookups, which are the file deps that are not captured and might result in different compilation mode. rdar://119275464 --- include/swift/ClangImporter/ClangImporter.h | 3 + lib/ClangImporter/ClangImporter.cpp | 138 ++++++++++++++---- .../ModuleDependencyScanner.cpp | 17 ++- lib/DependencyScan/ScanDependencies.cpp | 14 +- lib/Frontend/CompilerInvocation.cpp | 76 +++++----- lib/Frontend/ModuleInterfaceLoader.cpp | 28 +++- lib/Serialization/ScanningLoaders.cpp | 21 ++- test/CAS/Xcc_args.swift | 69 +++++++++ test/CAS/can-import.swift | 3 - test/CAS/include-tree.swift | 3 - test/CAS/module_deps.swift | 9 +- test/CAS/module_deps_include_tree.swift | 9 +- .../pcm-emit-direct-cc1-mode.swift | 3 +- 13 files changed, 290 insertions(+), 103 deletions(-) create mode 100644 test/CAS/Xcc_args.swift diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index 26b6748d5438b..e5fc22588da83 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -514,6 +514,9 @@ class ClangImporter final : public ClangModuleLoader { std::string getClangModuleHash() const; + /// Get clang import creation cc1 args for swift explicit module build. + std::vector getSwiftExplicitModuleDirectCC1Args() const; + /// If we already imported a given decl successfully, return the corresponding /// Swift decl as an Optional, but if we previously tried and failed /// to import said decl then return nullptr. diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 8ea8dae430de2..12e46176486b9 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -58,7 +58,9 @@ #include "clang/Basic/Module.h" #include "clang/Basic/TargetInfo.h" #include "clang/Basic/Version.h" +#include "clang/CAS/CASOptions.h" #include "clang/CodeGen/ObjectFilePCHContainerOperations.h" +#include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Frontend/Utils.h" @@ -73,8 +75,10 @@ #include "clang/Sema/Sema.h" #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/ASTWriter.h" +#include "clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/CrashRecoveryContext.h" #include "llvm/Support/FileCollector.h" @@ -1067,26 +1071,7 @@ std::optional> ClangImporter::getClangCC1Arguments( ClangImporter *importer, ASTContext &ctx, llvm::IntrusiveRefCntPtr VFS, bool ignoreClangTarget) { - // If using direct cc1 module build, return extra args only. - if (ctx.ClangImporterOpts.DirectClangCC1ModuleBuild) - return ctx.ClangImporterOpts.ExtraArgs; - - // Otherwise, create cc1 arguments from driver args. - auto driverArgs = getClangDriverArguments(ctx, ignoreClangTarget); - - llvm::SmallVector invocationArgs; - invocationArgs.reserve(driverArgs.size()); - llvm::for_each(driverArgs, [&](const std::string &Arg) { - invocationArgs.push_back(Arg.c_str()); - }); - - if (ctx.ClangImporterOpts.DumpClangDiagnostics) { - llvm::errs() << "clang importer driver args: '"; - llvm::interleave( - invocationArgs, [](StringRef arg) { llvm::errs() << arg; }, - [] { llvm::errs() << "' '"; }); - llvm::errs() << "'\n"; - } + std::unique_ptr CI; // Set up a temporary diagnostic client to report errors from parsing the // command line, which may be important for Swift clients if, for example, @@ -1098,24 +1083,60 @@ std::optional> ClangImporter::getClangCC1Arguments( // clang::CompilerInstance is created. llvm::IntrusiveRefCntPtr tempDiagOpts{ new clang::DiagnosticOptions}; - auto *tempDiagClient = new ClangDiagnosticConsumer(importer->Impl, *tempDiagOpts, ctx.ClangImporterOpts.DumpClangDiagnostics); - auto clangDiags = clang::CompilerInstance::createDiagnostics( tempDiagOpts.get(), tempDiagClient, /*owned*/ true); - clang::CreateInvocationOptions CIOpts; - CIOpts.VFS = VFS; - CIOpts.Diags = clangDiags; - CIOpts.RecoverOnError = false; - CIOpts.ProbePrecompiled = true; - auto CI = clang::createInvocation(invocationArgs, std::move(CIOpts)); + // If using direct cc1 module build, use extra args to setup ClangImporter. + if (ctx.ClangImporterOpts.DirectClangCC1ModuleBuild) { + llvm::SmallVector clangArgs; + clangArgs.reserve(ctx.ClangImporterOpts.ExtraArgs.size()); + llvm::for_each( + ctx.ClangImporterOpts.ExtraArgs, + [&](const std::string &Arg) { clangArgs.push_back(Arg.c_str()); }); + + // Try parse extra args, if failed, return nullopt. + CI = std::make_unique(); + if (!clang::CompilerInvocation::CreateFromArgs(*CI, clangArgs, + *clangDiags)) + return std::nullopt; - if (!CI) - return std::nullopt; + // Forwards some options from swift to clang even using direct mode. This is + // to reduce the number of argument passing on the command-line and swift + // compiler can be more efficient to compute swift cache key without having + // the knowledge about clang command-line options. + if (ctx.CASOpts.EnableCaching) + CI->getCASOpts() = ctx.CASOpts.CASOpts; + } else { + // Otherwise, create cc1 arguments from driver args. + auto driverArgs = getClangDriverArguments(ctx, ignoreClangTarget); + + llvm::SmallVector invocationArgs; + invocationArgs.reserve(driverArgs.size()); + llvm::for_each(driverArgs, [&](const std::string &Arg) { + invocationArgs.push_back(Arg.c_str()); + }); + + if (ctx.ClangImporterOpts.DumpClangDiagnostics) { + llvm::errs() << "clang importer driver args: '"; + llvm::interleave( + invocationArgs, [](StringRef arg) { llvm::errs() << arg; }, + [] { llvm::errs() << "' '"; }); + llvm::errs() << "'\n"; + } + + clang::CreateInvocationOptions CIOpts; + CIOpts.VFS = VFS; + CIOpts.Diags = clangDiags; + CIOpts.RecoverOnError = false; + CIOpts.ProbePrecompiled = true; + CI = clang::createInvocation(invocationArgs, std::move(CIOpts)); + if (!CI) + return std::nullopt; + } // FIXME: clang fails to generate a module if there is a `-fmodule-map-file` // argument pointing to a missing file. @@ -3919,6 +3940,63 @@ std::string ClangImporter::getClangModuleHash() const { return Impl.Invocation->getModuleHash(Impl.Instance->getDiagnostics()); } +std::vector +ClangImporter::getSwiftExplicitModuleDirectCC1Args() const { + llvm::SmallVector clangArgs; + clangArgs.reserve(Impl.ClangArgs.size()); + llvm::for_each(Impl.ClangArgs, [&](const std::string &Arg) { + clangArgs.push_back(Arg.c_str()); + }); + + clang::CompilerInvocation instance; + clang::DiagnosticsEngine clangDiags(new clang::DiagnosticIDs(), + new clang::DiagnosticOptions(), + new clang::IgnoringDiagConsumer()); + bool success = clang::CompilerInvocation::CreateFromArgs(instance, clangArgs, + clangDiags); + (void)success; + assert(success && "clang options from clangImporter failed to parse"); + + if (!Impl.SwiftContext.CASOpts.EnableCaching) + return instance.getCC1CommandLine(); + + // Clear some options that are not needed. + instance.clearImplicitModuleBuildOptions(); + + // CASOpts are forwarded from swift arguments. + instance.getCASOpts() = clang::CASOptions(); + + // HeaderSearchOptions. + // Clang search options are only used by scanner and clang importer from main + // module should not using search paths to find modules. + auto &HSOpts = instance.getHeaderSearchOpts(); + HSOpts.VFSOverlayFiles.clear(); + HSOpts.UserEntries.clear(); + HSOpts.SystemHeaderPrefixes.clear(); + + // FrontendOptions. + auto &FEOpts = instance.getFrontendOpts(); + FEOpts.IncludeTimestamps = false; + FEOpts.ModuleMapFiles.clear(); + + // PreprocessorOptions. + // Cannot clear macros as the main module clang importer doesn't have clang + // include tree created and it has to be created from command-line. However, + // include files are no collected into CASFS so they will not be found so + // clear them to avoid problem. + auto &PPOpts = instance.getPreprocessorOpts(); + PPOpts.MacroIncludes.clear(); + PPOpts.Includes.clear(); + + if (Impl.SwiftContext.ClangImporterOpts.UseClangIncludeTree) { + // FileSystemOptions. + auto &FSOpts = instance.getFileSystemOpts(); + FSOpts.WorkingDir.clear(); + } + + return instance.getCC1CommandLine(); +} + std::optional ClangImporter::importDeclCached(const clang::NamedDecl *ClangDecl) { return Impl.importDeclCached(ClangDecl, Impl.CurrentVersion); diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp index e16d5872d7a89..8a59384c14bae 100644 --- a/lib/DependencyScan/ModuleDependencyScanner.cpp +++ b/lib/DependencyScan/ModuleDependencyScanner.cpp @@ -346,6 +346,7 @@ ModuleDependencyScanner::getMainModuleDependencyInfo( {"-Xcc", "-target", "-Xcc", ScanASTContext.LangOpts.Target.str()}); std::string rootID; + std::vector buildArgs; if (tracker) { tracker->startTracking(); for (auto fileUnit : mainModule->getFiles()) { @@ -371,10 +372,22 @@ ModuleDependencyScanner::getMainModuleDependencyInfo( return std::make_error_code(std::errc::io_error); } rootID = root->getID().toString(); + + buildArgs.push_back("-direct-clang-cc1-module-build"); + for (auto &arg : clangImporter->getSwiftExplicitModuleDirectCC1Args()) { + buildArgs.push_back("-Xcc"); + buildArgs.push_back(arg); + } } - auto mainDependencies = - ModuleDependencyInfo::forSwiftSourceModule(rootID, {}, {}, ExtraPCMArgs); + llvm::SmallVector buildCommands; + buildCommands.reserve(buildArgs.size()); + llvm::for_each(buildArgs, [&](const std::string &arg) { + buildCommands.emplace_back(arg); + }); + + auto mainDependencies = ModuleDependencyInfo::forSwiftSourceModule( + rootID, buildCommands, {}, ExtraPCMArgs); llvm::StringSet<> alreadyAddedModules; // Compute Implicit dependencies of the main module diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index c82a27301a435..91afcbe23a9d0 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -290,19 +290,11 @@ static llvm::Error resolveExplicitModuleInputs( } } if (!clangDepDetails->moduleCacheKey.empty()) { - auto appendXclang = [&]() { - if (!resolvingDepInfo.isClangModule()) { - // clang module build using cc1 arg so this is not needed. - commandLine.push_back("-Xcc"); - commandLine.push_back("-Xclang"); - } - commandLine.push_back("-Xcc"); - }; - appendXclang(); + commandLine.push_back("-Xcc"); commandLine.push_back("-fmodule-file-cache-key"); - appendXclang(); + commandLine.push_back("-Xcc"); commandLine.push_back(clangDepDetails->mappedPCMPath); - appendXclang(); + commandLine.push_back("-Xcc"); commandLine.push_back(clangDepDetails->moduleCacheKey); } diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index e8f4512b7860b..5968c640421bc 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -1659,43 +1659,6 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts, ArgList &Args, Opts.ExtraArgs.push_back(A->getValue()); } - for (const Arg *A : Args.filtered(OPT_file_prefix_map, - OPT_debug_prefix_map)) { - std::string Val(A->getValue()); - // Forward -debug-prefix-map arguments from Swift to Clang as - // -fdebug-prefix-map= and -file-prefix-map as -ffile-prefix-map=. - // - // This is required to ensure DIFiles created there, like - /// "", as well as index data, have their paths - // remapped properly. - // - // (Note, however, that Clang's usage of std::map means that the remapping - // may not be applied in the same order, which can matter if one mapping is - // a prefix of another.) - if (A->getOption().matches(OPT_file_prefix_map)) - Opts.ExtraArgs.push_back("-ffile-prefix-map=" + Val); - else - Opts.ExtraArgs.push_back("-fdebug-prefix-map=" + Val); - } - - if (auto *A = Args.getLastArg(OPT_file_compilation_dir)) { - // Forward the -file-compilation-dir flag to correctly set the - // debug compilation directory. - std::string Val(A->getValue()); - Opts.ExtraArgs.push_back("-ffile-compilation-dir=" + Val); - } - - if (CASOpts.CASFSRootIDs.empty() && - CASOpts.ClangIncludeTrees.empty()) { - if (!workingDirectory.empty()) { - // Provide a working directory to Clang as well if there are any -Xcc - // options, in case some of them are search-related. But do it at the - // beginning, so that an explicit -Xcc -working-directory will win. - Opts.ExtraArgs.insert(Opts.ExtraArgs.begin(), - {"-working-directory", workingDirectory.str()}); - } - } - Opts.DumpClangDiagnostics |= Args.hasArg(OPT_dump_clang_diagnostics); // When the repl is invoked directly (ie. `lldb --repl="..."`) the action @@ -1755,6 +1718,45 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts, ArgList &Args, Opts.HasClangIncludeTreeRoot |= Args.hasArg(OPT_clang_include_tree_root); } + // If in direct clang cc1 module build mode, return early. + if (Opts.DirectClangCC1ModuleBuild) + return false; + + // Only amend the following path option when not in direct cc1 mode. + for (const Arg *A : Args.filtered(OPT_file_prefix_map, + OPT_debug_prefix_map)) { + std::string Val(A->getValue()); + // Forward -debug-prefix-map arguments from Swift to Clang as + // -fdebug-prefix-map= and -file-prefix-map as -ffile-prefix-map=. + // + // This is required to ensure DIFiles created there, like + /// "", as well as index data, have their paths + // remapped properly. + // + // (Note, however, that Clang's usage of std::map means that the remapping + // may not be applied in the same order, which can matter if one mapping is + // a prefix of another.) + if (A->getOption().matches(OPT_file_prefix_map)) + Opts.ExtraArgs.push_back("-ffile-prefix-map=" + Val); + else + Opts.ExtraArgs.push_back("-fdebug-prefix-map=" + Val); + } + + if (auto *A = Args.getLastArg(OPT_file_compilation_dir)) { + // Forward the -file-compilation-dir flag to correctly set the + // debug compilation directory. + std::string Val(A->getValue()); + Opts.ExtraArgs.push_back("-ffile-compilation-dir=" + Val); + } + + if (!workingDirectory.empty()) { + // Provide a working directory to Clang as well if there are any -Xcc + // options, in case some of them are search-related. But do it at the + // beginning, so that an explicit -Xcc -working-directory will win. + Opts.ExtraArgs.insert(Opts.ExtraArgs.begin(), + {"-working-directory", workingDirectory.str()}); + } + return false; } diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 7060f8922cf24..5da57096796f6 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -1675,6 +1675,9 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( Feature::LayoutPrespecialization); } + genericSubInvocation.getClangImporterOptions().DirectClangCC1ModuleBuild = + clangImporterOpts.DirectClangCC1ModuleBuild; + // Validate Clang modules once per-build session flags must be consistent // across all module sub-invocations if (clangImporterOpts.ValidateModulesOnce) { @@ -1690,6 +1693,8 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( genericSubInvocation.getCASOptions().CASOpts = casOpts.CASOpts; casOpts.enumerateCASConfigurationFlags( [&](StringRef Arg) { GenericArgs.push_back(ArgSaver.save(Arg)); }); + // ClangIncludeTree is default on when caching is enabled. + genericSubInvocation.getClangImporterOptions().UseClangIncludeTree = true; } if (!clangImporterOpts.UseClangIncludeTree) { @@ -2068,10 +2073,16 @@ InterfaceSubContextDelegateImpl::runInSubCompilerInstance(StringRef moduleName, == originalTargetTriple.getEnvironment()) { parsedTargetTriple.setArchName(originalTargetTriple.getArchName()); subInvocation.setTargetTriple(parsedTargetTriple.str()); + } - // Overload the target in the BuildArgs as well - BuildArgs.push_back("-target"); - BuildArgs.push_back(parsedTargetTriple.str()); + // Find and overload all "-target" to be parsedTargetTriple. This make sure + // the build command for the interface is the same no matter what the parent + // triple is so there is no need to spawn identical jobs. + assert(llvm::find(BuildArgs, "-target") != BuildArgs.end() && + "missing target option"); + for (unsigned idx = 0, end = BuildArgs.size(); idx < end; ++idx) { + if (BuildArgs[idx] == "-target" && ++idx < end) + BuildArgs[idx] = parsedTargetTriple.str(); } // restore `StrictImplicitModuleContext` @@ -2422,6 +2433,11 @@ struct ExplicitCASModuleLoader::Implementation { std::set moduleMapsSeen; std::vector &extraClangArgs = Ctx.ClangImporterOpts.ExtraArgs; + // Append -Xclang if we are not in direct cc1 mode. + auto appendXclang = [&]() { + if (!Ctx.ClangImporterOpts.DirectClangCC1ModuleBuild) + extraClangArgs.push_back("-Xclang"); + }; for (auto &entry : ExplicitClangModuleMap) { const auto &moduleMapPath = entry.getValue().moduleMapPath; if (!moduleMapPath.empty() && @@ -2440,11 +2456,11 @@ struct ExplicitCASModuleLoader::Implementation { } auto cachePath = entry.getValue().moduleCacheKey; if (cachePath) { - extraClangArgs.push_back("-Xclang"); + appendXclang(); extraClangArgs.push_back("-fmodule-file-cache-key"); - extraClangArgs.push_back("-Xclang"); + appendXclang(); extraClangArgs.push_back(modulePath); - extraClangArgs.push_back("-Xclang"); + appendXclang(); extraClangArgs.push_back(*cachePath); } } diff --git a/lib/Serialization/ScanningLoaders.cpp b/lib/Serialization/ScanningLoaders.cpp index cd1e12e484b35..eff9591515261 100644 --- a/lib/Serialization/ScanningLoaders.cpp +++ b/lib/Serialization/ScanningLoaders.cpp @@ -157,10 +157,23 @@ SwiftModuleScanner::scanInterfaceFile(Twine moduleInterfacePath, // Add explicit Swift dependency compilation flags Args.push_back("-explicit-interface-module-build"); Args.push_back("-disable-implicit-swift-modules"); - Args.push_back("-Xcc"); - Args.push_back("-fno-implicit-modules"); - Args.push_back("-Xcc"); - Args.push_back("-fno-implicit-module-maps"); + + // Handle clang arguments. For caching build, all arguments are passed + // with `-direct-clang-cc1-module-build`. + if (Ctx.CASOpts.EnableCaching) { + Args.push_back("-direct-clang-cc1-module-build"); + auto *importer = + static_cast(Ctx.getClangModuleLoader()); + for (auto &Arg : importer->getSwiftExplicitModuleDirectCC1Args()) { + Args.push_back("-Xcc"); + Args.push_back(Arg); + } + } else { + Args.push_back("-Xcc"); + Args.push_back("-fno-implicit-modules"); + Args.push_back("-Xcc"); + Args.push_back("-fno-implicit-module-maps"); + } for (const auto &candidate : compiledCandidates) { Args.push_back("-candidate-module-file"); Args.push_back(candidate); diff --git a/test/CAS/Xcc_args.swift b/test/CAS/Xcc_args.swift new file mode 100644 index 0000000000000..0ffbe0ace877d --- /dev/null +++ b/test/CAS/Xcc_args.swift @@ -0,0 +1,69 @@ +// RUN: %empty-directory(%t) +// RUN: split-file %s %t + +// RUN: %target-swift-frontend -scan-dependencies -module-name Test -module-cache-path %t/clang-module-cache -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: %t/test.swift -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -Xcc -D_VERSION=1 \ +// RUN: -Xcc -fmodule-map-file=%t/include/module.modulemap -Xcc -ivfsoverlay -Xcc %t/empty.yaml \ +// RUN: -Xcc -I%t/empty.hmap + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/shims.cmd +// RUN: %swift_frontend_plain @%t/shims.cmd + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Swift > %t/swift.cmd +// RUN: %swift_frontend_plain @%t/swift.cmd + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:_Macro > %t/Macro.cmd +// RUN: %swift_frontend_plain @%t/Macro.cmd + +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd +// RUN: %FileCheck %s --input-file=%t/MyApp.cmd + +// CHECK: "-direct-clang-cc1-module-build" +// CHECK: "_VERSION=1" +// CHECK-NOT: hmap.json +// CHECK-NOT: -ivfsoverlay +// CHECK-NOT: -fmodule-map-file + +// RUN: %target-swift-frontend \ +// RUN: -typecheck -cache-compile-job -cas-path %t/cas \ +// RUN: -swift-version 5 -disable-implicit-swift-modules \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name Test -explicit-swift-module-map-file @%t/map.casid \ +// RUN: %t/test.swift @%t/MyApp.cmd + +//--- test.swift +private import _Macro + +public func test() { + let _ = VERSION +} + +//--- include/module.modulemap +module _Macro { + umbrella "." + export * +} + +//--- include/Macro.h +#if defined(_VERSION) +#define VERSION _VERSION +#else +#define VERSION 0 +#endif + +//--- hmap.json +{ + "mappings": {} +} + +//--- empty.yaml +{ + "version": 0, + "case-sensitive": "false", + "use-external-names": true, + "roots": [] +} diff --git a/test/CAS/can-import.swift b/test/CAS/can-import.swift index 027fe378114eb..8483483cf0a3b 100644 --- a/test/CAS/can-import.swift +++ b/test/CAS/can-import.swift @@ -1,6 +1,3 @@ -// rdar://119964830 Temporarily disabling in Linux -// UNSUPPORTED: OS=linux-gnu - // RUN: %empty-directory(%t) // RUN: split-file %s %t diff --git a/test/CAS/include-tree.swift b/test/CAS/include-tree.swift index f3f0919385cb5..5a3334ed35865 100644 --- a/test/CAS/include-tree.swift +++ b/test/CAS/include-tree.swift @@ -1,6 +1,3 @@ -// rdar://119964830 Temporarily disabling in Linux -// UNSUPPORTED: OS=linux-gnu - // RUN: %empty-directory(%t) // RUN: split-file %s %t diff --git a/test/CAS/module_deps.swift b/test/CAS/module_deps.swift index a4115110478ba..49897a44bb970 100644 --- a/test/CAS/module_deps.swift +++ b/test/CAS/module_deps.swift @@ -31,9 +31,8 @@ // RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:F casFSRootID > %t/F_fs.casid // RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/F_fs.casid | %FileCheck %s -check-prefix FS_ROOT_F -/// make sure the number of CASFS is correct. 10 entries with 2 line each + 1 extra bracket -// RUN: NUM_CMDS=$(%{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps commandLine | wc -l) -// RUN: if [ ! $NUM_CMDS -eq 21 ]; then echo "wrong number of CASFS from scanning"; exit 1; fi +// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps commandLine > %t/deps.cmd +// RUN: %FileCheck %s -check-prefix MAIN_CMD -input-file=%t/deps.cmd // FS_ROOT_E-DAG: E.swiftinterface // FS_ROOT_E-DAG: SDKSettings.json @@ -49,6 +48,10 @@ // FS_ROOT_F: CHeaders/X.h // FS_ROOT_F: CHeaders/module.modulemap +// MAIN_CMD: -direct-clang-cc1-module-build +// MAIN_CMD: -cas-fs +// MAIN_CMD-NOT: -clang-include-tree-root + import C import E import G diff --git a/test/CAS/module_deps_include_tree.swift b/test/CAS/module_deps_include_tree.swift index d04a2e5f6057a..ed5afb4f02fc3 100644 --- a/test/CAS/module_deps_include_tree.swift +++ b/test/CAS/module_deps_include_tree.swift @@ -32,9 +32,8 @@ // RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:F clangIncludeTree > %t/F_tree.casid // RUN: clang-cas-test --cas %t/cas --print-include-tree @%t/F_tree.casid | %FileCheck %s -check-prefix INCLUDE_TREE_F -/// make sure the number of CASFS is correct. 10 entries with 2 line each + 1 extra bracket -// RUN: NUM_CMDS=$(%{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps commandLine | wc -l) -// RUN: if [ ! $NUM_CMDS -eq 21 ]; then echo "wrong number of CASFS from scanning"; exit 1; fi +// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps commandLine > %t/deps.cmd +// RUN: %FileCheck %s -check-prefix MAIN_CMD -input-file=%t/deps.cmd // FS_ROOT_E-DAG: E.swiftinterface // FS_ROOT_E-DAG: SDKSettings.json @@ -44,6 +43,10 @@ // INCLUDE_TREE_F: Files: // INCLUDE_TREE_F-NEXT: CHeaders/F.h +// MAIN_CMD: -direct-clang-cc1-module-build +// MAIN_CMD: -cas-fs +// MAIN_CMD: -clang-include-tree-root + import C import E import G diff --git a/test/ClangImporter/pcm-emit-direct-cc1-mode.swift b/test/ClangImporter/pcm-emit-direct-cc1-mode.swift index ff8d15bde050e..f5048b9e3d5fd 100644 --- a/test/ClangImporter/pcm-emit-direct-cc1-mode.swift +++ b/test/ClangImporter/pcm-emit-direct-cc1-mode.swift @@ -11,7 +11,8 @@ // Verify that the clang command-line used is cc1 // RUN: %FileCheck -check-prefix CHECK-CLANG -DTRIPLE=%target-triple %s < %t.diags.txt // CHECK-CLANG: clang importer cc1 args -// CHECK-CLANG-SAME: '{{.*[/\\]}}module.modulemap' '-o' '{{.*[/\\]}}script.pcm' '-fmodules' '-triple' '[[TRIPLE]]' '-x' 'objective-c' +// CHECK-CLANG-SAME: '-o' '{{.*[/\\]}}script.pcm' '-fsyntax-only' '-x' 'objective-c' '{{.*[/\\]}}module.modulemap' '-triple' '[[TRIPLE]]' +// CHECK-CLANG-SAME: '-fmodules' // CHECK-CLANG-NOT: clang importer driver args import script From 9f736811f0a6bdc87b5af81dce00929a48fb0673 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 15 Feb 2024 13:26:04 -0800 Subject: [PATCH 2/4] [swiftinterface] Improve target overwrite for the swiftinterface In certain cases (e.g. using arm64e interface to build arm64 target), the target needs to be updated when building swiftinterface. Push the target overwrite as early as possible to swiftinterface parsing by providing a preferred target to relevant functions. In such cases, the wrong target is never observed by other functions to avoid errors like the sub-invocation was partially setup for the wrong target. --- .../swift/Frontend/ModuleInterfaceLoader.h | 1 - .../Serialization/SerializedModuleLoader.h | 8 +-- lib/Frontend/ModuleInterfaceLoader.cpp | 51 +++++-------------- lib/Serialization/SerializedModuleLoader.cpp | 43 ++++++++-------- 4 files changed, 37 insertions(+), 66 deletions(-) diff --git a/include/swift/Frontend/ModuleInterfaceLoader.h b/include/swift/Frontend/ModuleInterfaceLoader.h index b1dab630c145e..4b7e0977759f0 100644 --- a/include/swift/Frontend/ModuleInterfaceLoader.h +++ b/include/swift/Frontend/ModuleInterfaceLoader.h @@ -643,7 +643,6 @@ struct InterfaceSubContextDelegateImpl: InterfaceSubContextDelegate { llvm::StringSaver ArgSaver; std::vector GenericArgs; CompilerInvocation genericSubInvocation; - llvm::Triple ParentInvocationTarget; template InFlightDiagnostic diagnose(StringRef interfacePath, diff --git a/include/swift/Serialization/SerializedModuleLoader.h b/include/swift/Serialization/SerializedModuleLoader.h index 70932b3ea3af2..b97a08938e0f3 100644 --- a/include/swift/Serialization/SerializedModuleLoader.h +++ b/include/swift/Serialization/SerializedModuleLoader.h @@ -18,6 +18,7 @@ #include "swift/AST/ModuleLoader.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/PrefixMapper.h" +#include "llvm/TargetParser/Triple.h" namespace swift { class ModuleFile; @@ -552,9 +553,10 @@ class SerializedASTFile final : public LoadedFile { }; /// Extract compiler arguments from an interface file buffer. -bool extractCompilerFlagsFromInterface(StringRef interfacePath, - StringRef buffer, llvm::StringSaver &ArgSaver, - SmallVectorImpl &SubArgs); +bool extractCompilerFlagsFromInterface( + StringRef interfacePath, StringRef buffer, llvm::StringSaver &ArgSaver, + SmallVectorImpl &SubArgs, + std::optional PreferredTarget = std::nullopt); /// Extract the user module version number from an interface file. llvm::VersionTuple extractUserModuleVersionFromInterface(StringRef moduleInterfacePath); diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 5da57096796f6..7278dde764ecf 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -1433,12 +1433,10 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface( SearchPathOpts.CandidateCompiledModules); } -static bool readSwiftInterfaceVersionAndArgs(SourceManager &SM, - DiagnosticEngine &Diags, - llvm::StringSaver &ArgSaver, - SwiftInterfaceInfo &interfaceInfo, - StringRef interfacePath, - SourceLoc diagnosticLoc) { +static bool readSwiftInterfaceVersionAndArgs( + SourceManager &SM, DiagnosticEngine &Diags, llvm::StringSaver &ArgSaver, + SwiftInterfaceInfo &interfaceInfo, StringRef interfacePath, + SourceLoc diagnosticLoc, llvm::Triple preferredTarget) { llvm::vfs::FileSystem &fs = *SM.getFileSystem(); auto FileOrError = swift::vfs::getFileOrSTDIN(fs, interfacePath); if (!FileOrError) { @@ -1461,7 +1459,8 @@ static bool readSwiftInterfaceVersionAndArgs(SourceManager &SM, } if (extractCompilerFlagsFromInterface(interfacePath, SB, ArgSaver, - interfaceInfo.Arguments)) { + interfaceInfo.Arguments, + preferredTarget)) { InterfaceSubContextDelegateImpl::diagnose( interfacePath, diagnosticLoc, SM, &Diags, diag::error_extracting_version_from_module_interface); @@ -1543,9 +1542,10 @@ bool ModuleInterfaceLoader::buildExplicitSwiftModuleFromSwiftInterface( llvm::BumpPtrAllocator alloc; llvm::StringSaver ArgSaver(alloc); SwiftInterfaceInfo InterfaceInfo; - readSwiftInterfaceVersionAndArgs(Instance.getSourceMgr(), Instance.getDiags(), - ArgSaver, InterfaceInfo, interfacePath, - SourceLoc()); + readSwiftInterfaceVersionAndArgs( + Instance.getSourceMgr(), Instance.getDiags(), ArgSaver, InterfaceInfo, + interfacePath, SourceLoc(), + Instance.getInvocation().getLangOptions().Target); auto Builder = ExplicitModuleInterfaceBuilder( Instance, &Instance.getDiags(), Instance.getSourceMgr(), @@ -1707,7 +1707,8 @@ bool InterfaceSubContextDelegateImpl::extractSwiftInterfaceVersionAndArgs( CompilerInvocation &subInvocation, SwiftInterfaceInfo &interfaceInfo, StringRef interfacePath, SourceLoc diagnosticLoc) { if (readSwiftInterfaceVersionAndArgs(SM, *Diags, ArgSaver, interfaceInfo, - interfacePath, diagnosticLoc)) + interfacePath, diagnosticLoc, + subInvocation.getLangOptions().Target)) return true; // Prior to Swift 5.9, swiftinterfaces were always built (accidentally) with @@ -1795,9 +1796,6 @@ InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl( GenericArgs.push_back("-application-extension"); } - // Save the parent invocation's Target Triple - ParentInvocationTarget = langOpts.Target; - // Pass down -explicit-swift-module-map-file StringRef explicitSwiftModuleMap = searchPathOpts.ExplicitSwiftModuleMap; genericSubInvocation.getSearchPathOptions().ExplicitSwiftModuleMap = @@ -2060,31 +2058,6 @@ InterfaceSubContextDelegateImpl::runInSubCompilerInstance(StringRef moduleName, BuildArgs.insert(BuildArgs.end(), interfaceInfo.Arguments.begin(), interfaceInfo.Arguments.end()); - // If the target triple parsed from the Swift interface file differs - // only in subarchitecture from the original target triple, then - // we have loaded a Swift interface from a different-but-compatible - // architecture slice. Use the original subarchitecture. - llvm::Triple parsedTargetTriple(subInvocation.getTargetTriple()); - if (parsedTargetTriple.getSubArch() != originalTargetTriple.getSubArch() && - parsedTargetTriple.getArch() == originalTargetTriple.getArch() && - parsedTargetTriple.getVendor() == originalTargetTriple.getVendor() && - parsedTargetTriple.getOS() == originalTargetTriple.getOS() && - parsedTargetTriple.getEnvironment() - == originalTargetTriple.getEnvironment()) { - parsedTargetTriple.setArchName(originalTargetTriple.getArchName()); - subInvocation.setTargetTriple(parsedTargetTriple.str()); - } - - // Find and overload all "-target" to be parsedTargetTriple. This make sure - // the build command for the interface is the same no matter what the parent - // triple is so there is no need to spawn identical jobs. - assert(llvm::find(BuildArgs, "-target") != BuildArgs.end() && - "missing target option"); - for (unsigned idx = 0, end = BuildArgs.size(); idx < end; ++idx) { - if (BuildArgs[idx] == "-target" && ++idx < end) - BuildArgs[idx] = parsedTargetTriple.str(); - } - // restore `StrictImplicitModuleContext` subInvocation.getFrontendOptions().StrictImplicitModuleContext = StrictImplicitModuleContext; diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp index eeeb2bac71594..5eaabe4c51c15 100644 --- a/lib/Serialization/SerializedModuleLoader.cpp +++ b/lib/Serialization/SerializedModuleLoader.cpp @@ -1265,10 +1265,10 @@ void swift::serialization::diagnoseSerializedASTLoadFailureTransitive( } } -bool swift::extractCompilerFlagsFromInterface(StringRef interfacePath, - StringRef buffer, - llvm::StringSaver &ArgSaver, - SmallVectorImpl &SubArgs) { +bool swift::extractCompilerFlagsFromInterface( + StringRef interfacePath, StringRef buffer, llvm::StringSaver &ArgSaver, + SmallVectorImpl &SubArgs, + std::optional PreferredTarget) { SmallVector FlagMatches; auto FlagRe = llvm::Regex("^// swift-module-flags:(.*)$", llvm::Regex::Newline); if (!FlagRe.match(buffer, &FlagMatches)) @@ -1276,26 +1276,23 @@ bool swift::extractCompilerFlagsFromInterface(StringRef interfacePath, assert(FlagMatches.size() == 2); llvm::cl::TokenizeGNUCommandLine(FlagMatches[1], ArgSaver, SubArgs); - auto intFileName = llvm::sys::path::filename(interfacePath); - - // Sanitize arch if the file name and the encoded flags disagree. - // It's a known issue that we are using arm64e interfaces contents for the arm64 target, - // meaning the encoded module flags are using -target arm64e-x-x. Fortunately, - // we can tell the target arch from the interface file name, so we could sanitize - // the target to use by inferring target from the file name. - StringRef arm64 = "arm64"; - StringRef arm64e = "arm64e"; - if (intFileName.contains(arm64) && !intFileName.contains(arm64e)) { - for (unsigned I = 1; I < SubArgs.size(); ++I) { - if (strcmp(SubArgs[I - 1], "-target") != 0) { - continue; - } - StringRef triple(SubArgs[I]); - if (triple.startswith(arm64e)) { - SubArgs[I] = ArgSaver.save((llvm::Twine(arm64) + - triple.substr(arm64e.size())).str()).data(); - } + // If the target triple parsed from the Swift interface file differs + // only in subarchitecture from the compatible target triple, then + // we have loaded a Swift interface from a different-but-compatible + // architecture slice. Use the compatible subarchitecture. + for (unsigned I = 1; I < SubArgs.size(); ++I) { + if (strcmp(SubArgs[I - 1], "-target") != 0) { + continue; } + llvm::Triple target(SubArgs[I]); + if (PreferredTarget && + target.getSubArch() != PreferredTarget->getSubArch() && + target.getArch() == PreferredTarget->getArch() && + target.getVendor() == PreferredTarget->getVendor() && + target.getOS() == PreferredTarget->getOS() && + target.getEnvironment() == PreferredTarget->getEnvironment()) + target.setArch(PreferredTarget->getArch(), PreferredTarget->getSubArch()); + SubArgs[I] = ArgSaver.save(target.str()).data(); } SmallVector IgnFlagMatches; From fb3c268abcc54c494faa8f9d27038f20275753b6 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 22 Feb 2024 14:45:42 -0800 Subject: [PATCH 3/4] [Caching] Support index while building when caching on Model indexing output as an optional output from the swift compiler as the build system has no knowledge about them and they can be regenerated by indexer. Make sure the indexing store output is produced when cache hit so the compilation is done for the module. If cache hit, no indexing data is produced since no compilation is done. rdar://123331335 --- include/swift/Option/Options.td | 2 +- lib/ClangImporter/ClangImporter.cpp | 7 ++++ test/CAS/index-store.swift | 55 +++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 test/CAS/index-store.swift diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index 257ad0204c7ae..14b5be52e240f 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -1454,7 +1454,7 @@ def index_file_path : Separate<["-"], "index-file-path">, MetaVarName<"">; def index_store_path : Separate<["-"], "index-store-path">, - Flags<[FrontendOption, ArgumentIsPath]>, MetaVarName<"">, + Flags<[FrontendOption, ArgumentIsPath, CacheInvariant]>, MetaVarName<"">, HelpText<"Store indexing data to ">; def index_unit_output_path : Separate<["-"], "index-unit-output-path">, diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 12e46176486b9..bda87ff4d7d24 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -1110,6 +1110,10 @@ std::optional> ClangImporter::getClangCC1Arguments( // the knowledge about clang command-line options. if (ctx.CASOpts.EnableCaching) CI->getCASOpts() = ctx.CASOpts.CASOpts; + + // Forward the index store path. That information is not passed to scanner + // and it is cached invariant so we don't want to re-scan if that changed. + CI->getFrontendOpts().IndexStorePath = ctx.ClangImporterOpts.IndexStorePath; } else { // Otherwise, create cc1 arguments from driver args. auto driverArgs = getClangDriverArguments(ctx, ignoreClangTarget); @@ -3979,6 +3983,9 @@ ClangImporter::getSwiftExplicitModuleDirectCC1Args() const { FEOpts.IncludeTimestamps = false; FEOpts.ModuleMapFiles.clear(); + // IndexStorePath is forwarded from swift. + FEOpts.IndexStorePath.clear(); + // PreprocessorOptions. // Cannot clear macros as the main module clang importer doesn't have clang // include tree created and it has to be created from command-line. However, diff --git a/test/CAS/index-store.swift b/test/CAS/index-store.swift new file mode 100644 index 0000000000000..5e39880edef21 --- /dev/null +++ b/test/CAS/index-store.swift @@ -0,0 +1,55 @@ +// RUN: %empty-directory(%t) +// RUN: split-file %s %t + +// RUN: %target-swift-frontend -emit-module %t/A.swift -I %t -module-name A \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \ +// RUN: -enable-library-evolution -swift-version 5 \ +// RUN: -emit-module-interface-path %t/A.swiftinterface \ +// RUN: -o %t/A.swiftmodule + +// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O -module-cache-path %t/clang-module-cache \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \ +// RUN: %t/test.swift -I %t -o %t/deps.json -cache-compile-job -cas-path %t/cas + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:B > %t/B.cmd +// RUN: %swift_frontend_plain @%t/B.cmd + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json A > %t/A.cmd +// RUN: %swift_frontend_plain @%t/A.cmd + +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd + +// RUN: %target-swift-frontend -cache-compile-job -Rcache-compile-job %t/test.swift -O -emit-module -o %t/Test.swiftmodule \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \ +// RUN: -disable-implicit-swift-modules -explicit-swift-module-map-file @%t/map.casid \ +// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd -index-system-modules -index-store-path %t/db 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s +// RUN: ls %t/db + +/// Cache hit with a different index-store-path. Note cache hit will skip replay index data. +// RUN: %target-swift-frontend -cache-compile-job -Rcache-compile-job %t/test.swift -O -emit-module -o %t/Test.swiftmodule \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \ +// RUN: -disable-implicit-swift-modules -explicit-swift-module-map-file @%t/map.casid \ +// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd -index-system-modules -index-store-path %t/db2 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s +// RUN: not ls %t/db2 + +// CACHE-MISS: remark: cache miss for input +// CACHE-HIT: remark: replay output file + +//--- test.swift +import A +import B +func test() {} + +//--- A.swift +func a() {} + +//--- module.modulemap +module B { + header "B.h" + export * +} + +//--- B.h +void b(void); From cb17ea8e6655b4659b51c35b844d55da873efd5a Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 23 Feb 2024 12:15:57 -0800 Subject: [PATCH 4/4] [ScanDependency] Add `-experimental-clang-importer-direct-cc1-scan` Add an experimental option to tell dependency scanner to report clang cc1 args should be used to construct clang importer in all constructed swift-frontend tasks. --- include/swift/Basic/LangOptions.h | 4 +++ include/swift/Option/Options.td | 5 ++++ .../ModuleDependencyScanner.cpp | 6 ++-- lib/Frontend/CompilerInvocation.cpp | 4 +++ lib/Frontend/ModuleInterfaceLoader.cpp | 2 ++ lib/Serialization/ScanningLoaders.cpp | 2 +- test/ScanDependencies/direct_cc1_scan.swift | 30 +++++++++++++++++++ 7 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 test/ScanDependencies/direct_cc1_scan.swift diff --git a/include/swift/Basic/LangOptions.h b/include/swift/Basic/LangOptions.h index 818e08400b899..e2960afc78b9c 100644 --- a/include/swift/Basic/LangOptions.h +++ b/include/swift/Basic/LangOptions.h @@ -969,6 +969,10 @@ namespace swift { /// Using ClangIncludeTreeRoot for compilation. bool HasClangIncludeTreeRoot = false; + /// Whether the dependency scanner should construct all swift-frontend + /// invocations directly from clang cc1 args. + bool ClangImporterDirectCC1Scan = false; + /// Return a hash code of any components from these options that should /// contribute to a Swift Bridging PCH hash. llvm::hash_code getPCHHashComponents() const { diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index 14b5be52e240f..2f0a2d05aa86b 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -1894,6 +1894,11 @@ def gcc_toolchain: Separate<["-"], "gcc-toolchain">, MetaVarName<"">, HelpText<"Specify a directory where the clang importer and clang linker can find headers and libraries">; +def experimental_clang_importer_direct_cc1_scan: + Flag<["-"], "experimental-clang-importer-direct-cc1-scan">, + Flags<[FrontendOption, NewDriverOnlyOption, HelpHidden]>, + HelpText<"Enables swift driver to construct swift-frontend invocations using -direct-clang-cc1-module-build">; + def cache_compile_job: Flag<["-"], "cache-compile-job">, Flags<[FrontendOption, NewDriverOnlyOption]>, HelpText<"Enable compiler caching">; diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp index 8a59384c14bae..f320ed46c3f98 100644 --- a/lib/DependencyScan/ModuleDependencyScanner.cpp +++ b/lib/DependencyScan/ModuleDependencyScanner.cpp @@ -347,6 +347,8 @@ ModuleDependencyScanner::getMainModuleDependencyInfo( std::string rootID; std::vector buildArgs; + auto clangImporter = + static_cast(ScanASTContext.getClangModuleLoader()); if (tracker) { tracker->startTracking(); for (auto fileUnit : mainModule->getFiles()) { @@ -359,8 +361,6 @@ ModuleDependencyScanner::getMainModuleDependencyInfo( ScanCompilerInvocation.getSearchPathOptions()); // Fetch some dependency files from clang importer. std::vector clangDependencyFiles; - auto clangImporter = - static_cast(ScanASTContext.getClangModuleLoader()); clangImporter->addClangInvovcationDependencies(clangDependencyFiles); llvm::for_each(clangDependencyFiles, [&](std::string &file) { tracker->trackFile(file); }); @@ -372,7 +372,9 @@ ModuleDependencyScanner::getMainModuleDependencyInfo( return std::make_error_code(std::errc::io_error); } rootID = root->getID().toString(); + } + if (ScanASTContext.ClangImporterOpts.ClangImporterDirectCC1Scan) { buildArgs.push_back("-direct-clang-cc1-module-build"); for (auto &arg : clangImporter->getSwiftExplicitModuleDirectCC1Args()) { buildArgs.push_back("-Xcc"); diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 5968c640421bc..705d9216fdd0f 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -1709,6 +1709,8 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts, ArgList &Args, Opts.DisableSourceImport |= Args.hasArg(OPT_disable_clangimporter_source_import); + Opts.ClangImporterDirectCC1Scan |= + Args.hasArg(OPT_experimental_clang_importer_direct_cc1_scan); // Forward the FrontendOptions to clang importer option so it can be // accessed when creating clang module compilation invocation. if (CASOpts.EnableCaching) { @@ -1716,6 +1718,8 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts, ArgList &Args, // useful in non-caching context. Opts.UseClangIncludeTree |= !Args.hasArg(OPT_no_clang_include_tree); Opts.HasClangIncludeTreeRoot |= Args.hasArg(OPT_clang_include_tree_root); + // Caching requires direct clang import cc1 scanning. + Opts.ClangImporterDirectCC1Scan = true; } // If in direct clang cc1 module build mode, return early. diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 7278dde764ecf..2b7d829ff5b45 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -1677,6 +1677,8 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( genericSubInvocation.getClangImporterOptions().DirectClangCC1ModuleBuild = clangImporterOpts.DirectClangCC1ModuleBuild; + genericSubInvocation.getClangImporterOptions().ClangImporterDirectCC1Scan = + clangImporterOpts.ClangImporterDirectCC1Scan; // Validate Clang modules once per-build session flags must be consistent // across all module sub-invocations diff --git a/lib/Serialization/ScanningLoaders.cpp b/lib/Serialization/ScanningLoaders.cpp index eff9591515261..a06452e359127 100644 --- a/lib/Serialization/ScanningLoaders.cpp +++ b/lib/Serialization/ScanningLoaders.cpp @@ -160,7 +160,7 @@ SwiftModuleScanner::scanInterfaceFile(Twine moduleInterfacePath, // Handle clang arguments. For caching build, all arguments are passed // with `-direct-clang-cc1-module-build`. - if (Ctx.CASOpts.EnableCaching) { + if (Ctx.ClangImporterOpts.ClangImporterDirectCC1Scan) { Args.push_back("-direct-clang-cc1-module-build"); auto *importer = static_cast(Ctx.getClangModuleLoader()); diff --git a/test/ScanDependencies/direct_cc1_scan.swift b/test/ScanDependencies/direct_cc1_scan.swift new file mode 100644 index 0000000000000..073d84836b9e3 --- /dev/null +++ b/test/ScanDependencies/direct_cc1_scan.swift @@ -0,0 +1,30 @@ +// RUN: %empty-directory(%t) +// RUN: split-file %s %t + +// RUN: %target-swift-frontend -emit-module %t/A.swift -module-name A \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \ +// RUN: -enable-library-evolution -swift-version 5 \ +// RUN: -emit-module-interface-path %t/A.swiftinterface \ +// RUN: -o %t/A.swiftmodule + +// RUN: %target-swift-frontend -scan-dependencies -o %t/deps.json -I %t \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \ +// RUN: %t/test.swift -module-name Test -swift-version 5 +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json A | %FileCheck %s --check-prefix CHECK-NO-DIRECT-CC1 +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json Test | %FileCheck %s --allow-empty --check-prefix CHECK-NO-DIRECT-CC1 + +// RUN: %target-swift-frontend -scan-dependencies -o %t/deps2.json -I %t \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \ +// RUN: %t/test.swift -module-name Test -swift-version 5 -experimental-clang-importer-direct-cc1-scan +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps2.json A | %FileCheck %s --check-prefix CHECK-DIRECT-CC1-SCAN +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps2.json Test | %FileCheck %s --check-prefix CHECK-DIRECT-CC1-SCAN + +// CHECK-NO-DIRECT-CC1-NOT: -direct-clang-cc1-module-build +// CHECK-DIRECT-CC1-SCAN: -direct-clang-cc1-module-build + +//--- A.swift +func a() {} + +//--- test.swift +import A +func test() {}