From daa1065304861375fd540b48f03f05a853c2a656 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 18 Dec 2023 15:53:26 -0800 Subject: [PATCH] [Caching] Preliminary simple macro support for caching Preliminary caching support for macro: * Inserting the plugin into the CASFS * Lookup plugin via physical file system For future better support, we should teach dependency scanner to resolve macros and return the resolved plugins to swift-frontend. rdar://121873571 --- lib/AST/ModuleDependencies.cpp | 53 ++++++++++++++++++++ lib/AST/PluginLoader.cpp | 16 ++++-- lib/Sema/TypeCheckMacros.cpp | 4 +- test/CAS/macro_option_set.swift | 53 ++++++++++++++++++++ test/CAS/macro_plugin.swift | 73 ++++++++++++++++++++++++++++ test/CAS/macro_plugin_external.swift | 73 ++++++++++++++++++++++++++++ 6 files changed, 268 insertions(+), 4 deletions(-) create mode 100644 test/CAS/macro_option_set.swift create mode 100644 test/CAS/macro_plugin.swift create mode 100644 test/CAS/macro_plugin_external.swift diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index b933181e48b0f..474fc1b689792 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -21,6 +21,7 @@ #include "swift/Frontend/Frontend.h" #include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" +#include "llvm/Config/config.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" #include "llvm/Support/PrefixMapper.h" @@ -481,6 +482,58 @@ void SwiftDependencyTracker::addCommonSearchPathDeps( // Add VFSOverlay file. for (auto &Overlay: Opts.VFSOverlayFiles) FS->status(Overlay); + + // Add plugin dylibs from the toolchain only by look through the plugin search + // directory. + auto recordFiles = [&](StringRef Path) { + std::error_code EC; + for (auto I = FS->dir_begin(Path, EC); + !EC && I != llvm::vfs::directory_iterator(); I = I.increment(EC)) { + if (I->type() != llvm::sys::fs::file_type::regular_file) + continue; +#if defined(_WIN32) + constexpr StringRef libPrefix{}; + constexpr StringRef libSuffix = ".dll"; +#else + constexpr StringRef libPrefix = "lib"; + constexpr StringRef libSuffix = LTDL_SHLIB_EXT; +#endif + StringRef filename = llvm::sys::path::filename(I->path()); + if (filename.starts_with(libPrefix) && filename.ends_with(libSuffix)) + FS->status(I->path()); + } + }; + for (auto &entry : Opts.PluginSearchOpts) { + switch (entry.getKind()) { + + // '-load-plugin-library '. + case PluginSearchOption::Kind::LoadPluginLibrary: { + auto &val = entry.get(); + FS->status(val.LibraryPath); + break; + } + + // '-load-plugin-executable #, ...'. + case PluginSearchOption::Kind::LoadPluginExecutable: { + // We don't have executable plugin in toolchain. + break; + } + + // '-plugin-path '. + case PluginSearchOption::Kind::PluginPath: { + auto &val = entry.get(); + recordFiles(val.SearchPath); + break; + } + + // '-external-plugin-path #'. + case PluginSearchOption::Kind::ExternalPluginPath: { + auto &val = entry.get(); + recordFiles(val.SearchPath); + break; + } + } + } } void SwiftDependencyTracker::startTracking() { diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index 658b28348e1d4..95f1b01ed24a8 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -17,6 +17,7 @@ #include "swift/Basic/SourceManager.h" #include "swift/Parse/Lexer.h" #include "llvm/Config/config.h" +#include "llvm/Support/VirtualFileSystem.h" using namespace swift; @@ -59,6 +60,15 @@ static StringRef pluginModuleNameStringFromPath(StringRef path) { return ""; } +static llvm::IntrusiveRefCntPtr +getPluginLoadingFS(ASTContext &Ctx) { + // If there is a clang include tree FS, using real file system to load plugin + // as the FS in SourceMgr doesn't support directory iterator. + if (Ctx.ClangImporterOpts.HasClangIncludeTreeRoot) + return llvm::vfs::getRealFileSystem(); + return Ctx.SourceMgr.getFileSystem(); +} + llvm::DenseMap & PluginLoader::getPluginMap() { if (PluginMap.has_value()) { @@ -86,7 +96,7 @@ PluginLoader::getPluginMap() { (void)result; }; - auto fs = Ctx.SourceMgr.getFileSystem(); + auto fs = getPluginLoadingFS(Ctx); std::error_code ec; for (auto &entry : Ctx.SearchPathOpts.PluginSearchOpts) { @@ -162,7 +172,7 @@ PluginLoader::lookupPluginByModuleName(Identifier moduleName) { llvm::Expected PluginLoader::loadLibraryPlugin(StringRef path) { - auto fs = Ctx.SourceMgr.getFileSystem(); + auto fs = getPluginLoadingFS(Ctx); SmallString<128> resolvedPath; if (auto err = fs->getRealPath(path, resolvedPath)) { return llvm::createStringError(err, err.message()); @@ -186,7 +196,7 @@ PluginLoader::loadLibraryPlugin(StringRef path) { llvm::Expected PluginLoader::loadExecutablePlugin(StringRef path) { - auto fs = Ctx.SourceMgr.getFileSystem(); + auto fs = getPluginLoadingFS(Ctx); SmallString<128> resolvedPath; if (auto err = fs->getRealPath(path, resolvedPath)) { return llvm::createStringError(err, err.message()); diff --git a/lib/Sema/TypeCheckMacros.cpp b/lib/Sema/TypeCheckMacros.cpp index 84a0d273f150f..efdcc35414c02 100644 --- a/lib/Sema/TypeCheckMacros.cpp +++ b/lib/Sema/TypeCheckMacros.cpp @@ -290,7 +290,9 @@ initializeExecutablePlugin(ASTContext &ctx, if (!libraryPath.empty()) { #if SWIFT_BUILD_SWIFT_SYNTAX llvm::SmallString<128> resolvedLibraryPath; - auto fs = ctx.SourceMgr.getFileSystem(); + auto fs = ctx.ClangImporterOpts.HasClangIncludeTreeRoot + ? llvm::vfs::getRealFileSystem() + : ctx.SourceMgr.getFileSystem(); if (auto err = fs->getRealPath(libraryPath, resolvedLibraryPath)) { return llvm::createStringError(err, err.message()); } diff --git a/test/CAS/macro_option_set.swift b/test/CAS/macro_option_set.swift new file mode 100644 index 0000000000000..f2e95a4dd749b --- /dev/null +++ b/test/CAS/macro_option_set.swift @@ -0,0 +1,53 @@ +// REQUIRES: swift_swift_parser + +// RUN: %empty-directory(%t) + +// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -plugin-path %swift-plugin-dir + +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid +// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS + +// FS: SwiftMacros + +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/SwiftShims.cmd +// RUN: %swift_frontend_plain @%t/SwiftShims.cmd + +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %target-swift-frontend \ +// RUN: -typecheck -verify -cache-compile-job -cas-path %t/cas \ +// RUN: -swift-version 5 -disable-implicit-swift-modules \ +// RUN: -plugin-path %swift-plugin-dir \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \ +// RUN: %s @%t/MyApp.cmd + +import Swift + +@attached(member, names: named(RawValue), named(rawValue), named(`init`), arbitrary) +@attached(extension, conformances: OptionSet) +public macro OptionSet() = + #externalMacro(module: "SwiftMacros", type: "OptionSetMacro") + +@OptionSet +struct ShippingOptions { + private enum Options: Int { + case nextDay + case secondDay + case priority + case standard + } + + static let express: ShippingOptions = [.nextDay, .secondDay] + static let all: ShippingOptions = [.express, .priority, .standard] +} + +let options = ShippingOptions.express +assert(options.contains(.nextDay)) +assert(options.contains(.secondDay)) +assert(!options.contains(.standard)) + diff --git a/test/CAS/macro_plugin.swift b/test/CAS/macro_plugin.swift new file mode 100644 index 0000000000000..71c26e32b675e --- /dev/null +++ b/test/CAS/macro_plugin.swift @@ -0,0 +1,73 @@ +// REQUIRES: swift_swift_parser + +/// Test loading and external library through `-load-plugin-library` +/// TODO: switch this test case to use `-external-plugin-path`. + +// RUN: %empty-directory(%t) +// RUN: %empty-directory(%t/plugins) +// +//== Build the plugin library +// RUN: %host-build-swift \ +// RUN: -swift-version 5 \ +// RUN: -emit-library \ +// RUN: -o %t/plugins/%target-library-name(MacroDefinition) \ +// RUN: -module-name=MacroDefinition \ +// RUN: %S/../Macros/Inputs/syntax_macro_definitions.swift \ +// RUN: -g -no-toolchain-stdlib-rpath + +// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -load-plugin-library %t/plugins/%target-library-name(MacroDefinition) + +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid +// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS + +// FS: MacroDefinition + +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/SwiftShims.cmd +// RUN: %swift_frontend_plain @%t/SwiftShims.cmd + +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %target-swift-frontend \ +// RUN: -typecheck -verify -cache-compile-job -cas-path %t/cas \ +// RUN: -swift-version 5 -disable-implicit-swift-modules \ +// RUN: -load-plugin-library %t/plugins/%target-library-name(MacroDefinition) \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \ +// RUN: %s @%t/MyApp.cmd + +@attached(extension, conformances: P, names: named(requirement)) +macro DelegatedConformance() = #externalMacro(module: "MacroDefinition", type: "DelegatedConformanceViaExtensionMacro") + +protocol P { + static func requirement() +} + +struct Wrapped: P { + static func requirement() { + print("Wrapped.requirement") + } +} + +@DelegatedConformance +struct Generic {} + +// CHECK: {"expandMacroResult":{"diagnostics":[],"expandedSource":"extension Generic: P where Element: P {\n static func requirement() {\n Element.requirement()\n }\n}"}} + +func requiresP(_ value: (some P).Type) { + value.requirement() +} + +requiresP(Generic.self) + +struct Outer { + @DelegatedConformance + struct Nested {} +} + +// CHECK: {"expandMacroResult":{"diagnostics":[],"expandedSource":"extension Outer.Nested: P where Element: P {\n static func requirement() {\n Element.requirement()\n }\n}"}} + +requiresP(Outer.Nested.self) diff --git a/test/CAS/macro_plugin_external.swift b/test/CAS/macro_plugin_external.swift new file mode 100644 index 0000000000000..7a904f4d3172a --- /dev/null +++ b/test/CAS/macro_plugin_external.swift @@ -0,0 +1,73 @@ +// REQUIRES: swift_swift_parser + +/// Test loading and external library through `-load-plugin-library` +/// TODO: switch this test case to use `-external-plugin-path`. + +// RUN: %empty-directory(%t) +// RUN: %empty-directory(%t/plugins) +// +//== Build the plugin library +// RUN: %host-build-swift \ +// RUN: -swift-version 5 \ +// RUN: -emit-library \ +// RUN: -o %t/plugins/%target-library-name(MacroDefinition) \ +// RUN: -module-name=MacroDefinition \ +// RUN: %S/../Macros/Inputs/syntax_macro_definitions.swift \ +// RUN: -g -no-toolchain-stdlib-rpath + +// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server + +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid +// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS + +// FS: MacroDefinition + +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/SwiftShims.cmd +// RUN: %swift_frontend_plain @%t/SwiftShims.cmd + +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %target-swift-frontend \ +// RUN: -typecheck -verify -cache-compile-job -cas-path %t/cas \ +// RUN: -swift-version 5 -disable-implicit-swift-modules \ +// RUN: -external-plugin-path %t/plugins/#%swift-plugin-server \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \ +// RUN: %s @%t/MyApp.cmd + +@attached(extension, conformances: P, names: named(requirement)) +macro DelegatedConformance() = #externalMacro(module: "MacroDefinition", type: "DelegatedConformanceViaExtensionMacro") + +protocol P { + static func requirement() +} + +struct Wrapped: P { + static func requirement() { + print("Wrapped.requirement") + } +} + +@DelegatedConformance +struct Generic {} + +// CHECK: {"expandMacroResult":{"diagnostics":[],"expandedSource":"extension Generic: P where Element: P {\n static func requirement() {\n Element.requirement()\n }\n}"}} + +func requiresP(_ value: (some P).Type) { + value.requirement() +} + +requiresP(Generic.self) + +struct Outer { + @DelegatedConformance + struct Nested {} +} + +// CHECK: {"expandMacroResult":{"diagnostics":[],"expandedSource":"extension Outer.Nested: P where Element: P {\n static func requirement() {\n Element.requirement()\n }\n}"}} + +requiresP(Outer.Nested.self)