[LTO] Setup LTO pipeline and swift-lto tool

include/swift/AST/DiagnosticsFrontend.def

@@ -358,6 +358,10 @@ REMARK(interface_file_lock_failure,none,
 
 REMARK(interface_file_lock_timed_out,none,
       "timed out waiting to acquire lock file for module interface '%0'", (StringRef))
+ERROR(invalid_serialized_module,none,
+      "unable to register invalid serialized module", ())
+ERROR(unable_to_load_serialized_module,none,
+      "unable to load serialized module '%0'", (StringRef))
 
 // Dependency Verifier Diagnostics
 ERROR(dependency_cascading_mismatch,none,

include/swift/LTO/LTO.h

@@ -0,0 +1,71 @@
+//===--- LTO.cpp - Swift LTO ----------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_LTO_H
+#define SWIFT_LTO_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <functional>
+#include <memory>
+
+#include "swift/AST/ASTContext.h"
+#include "swift/AST/IRGenOptions.h"
+#include "swift/ClangImporter/ClangImporterOptions.h"
+#include "swift/Frontend/ModuleInterfaceLoader.h"
+#include "swift/Frontend/PrintingDiagnosticConsumer.h"
+#include "swift/Serialization/Validation.h"
+
+namespace swift {
+
+class ASTContext;
+
+namespace lto {
+
+using GetStreamFn = std::function<std::unique_ptr<llvm::raw_ostream>(
+    llvm::StringRef ModuleName)>;
+
+class LTOPipeline {
+  llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryPaths;
+  llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryImportPaths;
+  llvm::StringRef RuntimeResourcePath;
+  llvm::SmallVector<Identifier, 2> ModuleNames;
+  LangOptions LangOpts;
+  ClangImporterOptions ClangOpts;
+  TypeCheckerOptions TCOpts;
+  SearchPathOptions SearchPathOpts;
+  SourceManager SM;
+  DiagnosticEngine Diags;
+  PrintingDiagnosticConsumer PrintDiags;
+  std::unique_ptr<ASTContext> Ctx;
+  MemoryBufferSerializedModuleLoader *MBL;
+
+public:
+  LTOPipeline(llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryPaths,
+              llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryImportPaths,
+              llvm::StringRef RuntimeResourcePath)
+      : RuntimeLibraryPaths(RuntimeLibraryPaths),
+        RuntimeLibraryImportPaths(RuntimeLibraryImportPaths),
+        RuntimeResourcePath(RuntimeResourcePath), Diags(SM) {}
+  bool addModule(std::unique_ptr<llvm::MemoryBuffer> Buffer);
+  bool emitLLVMModules(GetStreamFn GetStream);
+
+private:
+  ASTContext *createASTContext(serialization::ValidationInfo info,
+                               serialization::ExtendedValidationInfo extInfo);
+};
+
+} // namespace lto
+} // namespace swift
+
+#endif // SWIFT_LTO_H

lib/CMakeLists.txt

@@ -27,6 +27,7 @@ add_subdirectory(IDE)
 add_subdirectory(Immediate)
 add_subdirectory(IRGen)
 add_subdirectory(LLVMPasses)
+add_subdirectory(LTO)
 add_subdirectory(Markup)
 add_subdirectory(Migrator)
 add_subdirectory(Option)

lib/FrontendTool/FrontendTool.cpp

@@ -1384,6 +1384,7 @@ static bool serializeSIB(SILModule *SM, const PrimarySpecificPaths &PSPs,
   serializationOpts.OutputPath = moduleOutputPath.c_str();
   serializationOpts.SerializeAllSIL = true;
   serializationOpts.IsSIB = true;
+  serializationOpts.SerializeOptionsForDebugging = true;
 
   serialize(MSF, serializationOpts, SM);
   return Context.hadError();

lib/LTO/CMakeLists.txt

@@ -0,0 +1,7 @@
+add_swift_host_library(swiftLTO STATIC
+  LTO.cpp)
+target_link_libraries(swiftLTO PRIVATE
+  swiftFrontend
+  swiftIRGen
+  swiftSILGen
+  swiftSILOptimizer)

lib/LTO/LTO.cpp

@@ -0,0 +1,122 @@
+//===--- LTO.cpp - Swift LTO ----------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "swift/LTO/LTO.h"
+#include "swift/AST/DiagnosticsFrontend.h"
+#include "swift/AST/IRGenRequests.h"
+#include "swift/ClangImporter/ClangImporter.h"
+#include "swift/SIL/SILModule.h"
+#include "swift/Serialization/SerializedModuleLoader.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/IR/Module.h"
+
+namespace swift {
+
+namespace lto {
+
+using namespace llvm;
+
+bool LTOPipeline::addModule(std::unique_ptr<MemoryBuffer> Buffer) {
+  serialization::ExtendedValidationInfo extendedInfo;
+  serialization::ValidationInfo info =
+      serialization::validateSerializedAST(Buffer->getBuffer(), &extendedInfo);
+  if (info.status != serialization::Status::Valid) {
+    Diags.diagnose(SourceLoc(), diag::invalid_serialized_module);
+    return true;
+  }
+
+  if (!Ctx) {
+    Ctx.reset(createASTContext(info, extendedInfo));
+  }
+
+  MBL->registerMemoryBuffer(info.name, std::move(Buffer));
+
+  ModuleNames.emplace_back(Ctx->getIdentifier(info.name));
+  return false;
+}
+
+bool LTOPipeline::emitLLVMModules(GetStreamFn GetStream) {
+  IRGenOptions Opts = {};
+  Opts.OutputKind = IRGenOutputKind::Module;
+
+  for (auto &ModuleName : ModuleNames) {
+    std::vector<swift::Located<swift::Identifier>> AccessPath;
+    AccessPath.emplace_back(ModuleName, SourceLoc());
+    auto SwiftModule = Ctx->getModule(AccessPath);
+    if (!SwiftModule) {
+      Diags.diagnose(SourceLoc(), diag::unable_to_load_serialized_module,
+                     ModuleName.get());
+      return true;
+    }
+    Lowering::TypeConverter Types(*SwiftModule);
+    SILOptions SILOpts = {};
+    auto SM = performASTLowering(SwiftModule, Types, SILOpts);
+    // TODO: Propagate input file name through SIB to enable debug info
+    const PrimarySpecificPaths PSPs;
+    auto GeneratedMod =
+        performIRGeneration(Opts, SM->getSwiftModule(), std::move(SM),
+                            ModuleName.get(), PSPs, ArrayRef<std::string>());
+    auto LLVMMod = GeneratedMod.getModule();
+    if (auto OS = GetStream(LLVMMod->getName())) {
+      WriteBitcodeToFile(*LLVMMod, *OS);
+    }
+  }
+  return false;
+}
+
+ASTContext *
+LTOPipeline::createASTContext(serialization::ValidationInfo info,
+                              serialization::ExtendedValidationInfo extInfo) {
+  auto Ctx = ASTContext::get(LangOpts, TCOpts, SearchPathOpts, SM, Diags);
+  Diags.addConsumer(PrintDiags);
+  LangOpts.setTarget(Triple(info.targetTriple));
+  SearchPathOpts.SDKPath = extInfo.getSDKPath();
+
+  SearchPathOpts.RuntimeLibraryPaths.insert(
+      SearchPathOpts.RuntimeLibraryPaths.end(), RuntimeLibraryPaths.begin(),
+      RuntimeLibraryPaths.end());
+  SearchPathOpts.RuntimeLibraryImportPaths.insert(
+      SearchPathOpts.RuntimeLibraryImportPaths.end(),
+      RuntimeLibraryImportPaths.begin(), RuntimeLibraryImportPaths.end());
+  SearchPathOpts.RuntimeResourcePath = RuntimeResourcePath;
+
+  // MARK: Setup module loaders
+  std::unique_ptr<ClangImporter> clangImporter =
+      ClangImporter::create(*Ctx, ClangOpts, "", nullptr);
+  auto const &Clang = clangImporter->getClangInstance();
+  std::string ModuleCachePath = getModuleCachePathFromClang(Clang);
+
+  auto MIL = ModuleInterfaceLoader::create(*Ctx, ModuleCachePath, "", nullptr,
+                                           ModuleLoadingMode::PreferSerialized);
+  Ctx->addModuleLoader(std::move(MIL));
+  auto MBL = MemoryBufferSerializedModuleLoader::create(
+      *Ctx, nullptr, ModuleLoadingMode::OnlySerialized, true);
+  this->MBL = MBL.get();
+
+  auto SML = SerializedModuleLoader::create(
+      *Ctx, nullptr, ModuleLoadingMode::OnlySerialized, true);
+
+  Ctx->addModuleLoader(std::move(MBL));
+  Ctx->addModuleLoader(std::move(SML));
+  Ctx->addModuleLoader(std::move(clangImporter), /*isClang*/ true);
+
+  registerIRGenRequestFunctions(Ctx->evaluator);
+  registerSILOptimizerRequestFunctions(Ctx->evaluator);
+  registerParseRequestFunctions(Ctx->evaluator);
+  registerTypeCheckerRequestFunctions(Ctx->evaluator);
+  registerSILGenRequestFunctions(Ctx->evaluator);
+  registerIRGenSILTransforms(*Ctx);
+  return Ctx;
+}
+
+} // namespace lto
+} // namespace swift

test/LTO/Inputs/lib.swift

@@ -0,0 +1,3 @@
+public struct LibX {}
+
+public func getLibX() -> LibX { return LibX() }

test/LTO/Inputs/main.swift

@@ -0,0 +1,3 @@
+import lib
+
+_ = getLibX()

test/LTO/pipeline.swift

@@ -0,0 +1,24 @@
+// RUN: %empty-directory(%t)
+// RUN: cd %t && %target-swiftc_driver -emit-module %S/Inputs/lib.swift
+// RUN: cd %t && %target-swift-frontend -emit-sib %S/Inputs/lib.swift -parse-as-library
+// RUN: cd %t && %target-swift-frontend -emit-sib -I%t  %S/Inputs/main.swift
+
+// Examine loading order
+// RUN: cd %t && %swift-lto main.sib lib.sib
+// RUN: cd %t && %llvm-dis lib.bc -o - | %FileCheck %s -check-prefix=CHECK-LIB
+// RUN: cd %t && %llvm-dis main.bc -o - | %FileCheck %s -check-prefix=CHECK-MAIN
+
+// RUN: cd %t && %swift-lto lib.sib main.sib
+// RUN: cd %t && %llvm-dis lib.bc -o - | %FileCheck %s -check-prefix=CHECK-LIB
+// RUN: cd %t && %llvm-dis main.bc -o - | %FileCheck %s -check-prefix=CHECK-MAIN
+
+// CHECK-LIB: ModuleID = 'lib.bc'
+// CHECK-LIB: define hidden swiftcc void @"$s3lib4LibXVACycfC"()
+// CHECK-LIB: define swiftcc void @"$s3lib7getLibXAA0C1XVyF"()
+// CHECK-LIB:   call swiftcc void @"$s3lib4LibXVACycfC"
+
+// CHECK-MAIN: ModuleID = 'main.bc'
+// CHECK-MAIN: define i32 @main
+// CHECK-MAIN:   call swiftcc void @"$s3lib7getLibXAA0C1XVyF"
+// CHECK-MAIN: declare swiftcc void @"$s3lib7getLibXAA0C1XVyF"()
+

test/lit.cfg

@@ -398,6 +398,7 @@ if '-disable-astscope-lookup' in config.swift_test_options:
 if '-disable-parser-lookup' in config.swift_test_options:
     config.available_features.add("disable-parser-lookup")
 config.substitutions.append( ('%swift-indent', config.swift_indent) )
+config.substitutions.append( ('%swift-lto', '%s lto' % config.sil_llvm_gen) )
 config.substitutions.append( ('%llvm-link', config.llvm_link) )
 config.substitutions.append( ('%swift-llvm-opt', config.swift_llvm_opt) )
 config.substitutions.append( ('%llvm-dwarfdump', config.llvm_dwarfdump) )

tools/sil-llvm-gen/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_swift_host_tool(sil-llvm-gen
   SILLLVMGen.cpp
+  swift_lto_main.cpp
   SWIFT_COMPONENT tools
 )
 target_link_libraries(sil-llvm-gen
@@ -8,6 +9,7 @@ target_link_libraries(sil-llvm-gen
                         swiftIRGen
                         swiftSILGen
                         swiftSILOptimizer
+                        swiftLTO
                         # Clang libraries included to appease the linker on linux.
                         clangBasic
                         clangCodeGen)

tools/sil-llvm-gen/SILLLVMGen.cpp

@@ -111,12 +111,19 @@ static llvm::cl::opt<bool>
 // without being given the address of a function in the main executable).
 void anchorForGetMainExecutable() {}
 
+extern llvm::cl::SubCommand LTOSubcommand;
+int swift_lto_main(int argc, char **argv);
+
 int main(int argc, char **argv) {
   PROGRAM_START(argc, argv);
   INITIALIZE_LLVM();
 
   llvm::cl::ParseCommandLineOptions(argc, argv, "Swift LLVM IR Generator\n");
 
+  if (LTOSubcommand) {
+    return swift_lto_main(argc, argv);
+  }
+
   if (PrintStats)
     llvm::EnableStatistics();
 

tools/sil-llvm-gen/swift_lto_main.cpp

@@ -0,0 +1,90 @@
+//===--- swift_lto_main.cpp -----------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// This is a tool for reading sib files and running LTO passes upon them.
+///
+//===----------------------------------------------------------------------===//
+
+#include "swift/Basic/LLVMInitialize.h"
+#include "swift/Frontend/Frontend.h"
+#include "swift/LTO/LTO.h"
+#include "swift/Strings.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <cstdio>
+#include <memory>
+using namespace swift;
+
+llvm::cl::SubCommand LTOSubcommand("lto", "Swift LTO Tool");
+
+static llvm::cl::list<std::string>
+    InputFilenames(llvm::cl::Positional, llvm::cl::desc("[input files...]"),
+                   llvm::cl::OneOrMore, llvm::cl::sub(LTOSubcommand));
+
+// This function isn't referenced outside its translation unit, but it
+// can't use the "static" keyword because its address is used for
+// getMainExecutable (since some platforms don't support taking the
+// address of main, and some platforms can't implement getMainExecutable
+// without being given the address of a function in the main executable).
+void anchorForGetMainExecutableInSwiftLTO() {}
+
+// This tool is combined with sil-llvm-gen to reduce link time.
+// This entrypoint is invoked from SILLLVMGen.cpp when user invoke
+// lto subcommand.
+int swift_lto_main(int argc, char **argv) {
+  CompilerInvocation Invocation;
+
+  Invocation.setMainExecutablePath(llvm::sys::fs::getMainExecutable(
+      argv[0],
+      reinterpret_cast<void *>(&anchorForGetMainExecutableInSwiftLTO)));
+
+  auto SearchPathOpts = Invocation.getSearchPathOptions();
+  llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryPaths;
+  llvm::SmallVector<llvm::StringRef, 4> RuntimeLibraryImportPaths;
+  RuntimeLibraryPaths.insert(RuntimeLibraryPaths.begin(),
+                             SearchPathOpts.RuntimeLibraryPaths.begin(),
+                             SearchPathOpts.RuntimeLibraryPaths.end());
+  RuntimeLibraryImportPaths.insert(
+      RuntimeLibraryImportPaths.begin(),
+      SearchPathOpts.RuntimeLibraryImportPaths.begin(),
+      SearchPathOpts.RuntimeLibraryImportPaths.end());
+  lto::LTOPipeline Pipeline(RuntimeLibraryPaths, RuntimeLibraryImportPaths,
+                            SearchPathOpts.RuntimeResourcePath);
+
+  for (auto InputFilename : InputFilenames) {
+    // Load the input file.
+    auto FileBufOrErr = llvm::MemoryBuffer::getFileOrSTDIN(InputFilename);
+    if (!FileBufOrErr) {
+      fprintf(stderr, "Error! Failed to open file: %s\n",
+              InputFilename.c_str());
+      exit(-1);
+    }
+
+    if (Pipeline.addModule(std::move(FileBufOrErr.get()))) {
+      fprintf(stderr, "Error! Failed to load serialized module: %s\n",
+              InputFilename.c_str());
+      exit(-1);
+    }
+  }
+
+  Pipeline.emitLLVMModules([&](StringRef ModuleName) {
+    std::error_code EC;
+    std::unique_ptr<llvm::raw_ostream> RawOS =
+        std::make_unique<llvm::raw_fd_ostream>(ModuleName.str() + ".bc", EC);
+    if (EC)
+      return std::unique_ptr<llvm::raw_ostream>(nullptr);
+    return RawOS;
+  });
+  return 0;
+}