[Driver][SYCL] Enable ability to consume fat objects containing SPIR-V (#5251)

* [Driver][SYCL] Enable ability to consume fat objects containing SPIR-V

When a user creates objects/archives based off of -fsycl-device-obj=spirv
we want to be able to consume these objects seamlessly.  After any objects
are unbundled, we will pass these through spirv-to-ir-wrapper to potentially
convert them to LLVM-IR before performing the device link.

Toolchain flow will resemble:

          fat object
              |
           unbundle
              |
      +-------+------+
      |              |
  spirv-to-ir        |
      |              |
  llvm-link          |
      |              |
    wrap             |
      |              |
      +-------+------+
              |
             link

* [NFC] update to use new tool name

* clang format

* Update comment and adjust wrapper call from review feedback

* Adjust tests to check for llvm-foreach call with spirv-to-ir-wrapper

* fix comment newline

Author: mdtoguchi

clang/include/clang/Driver/Action.h

@@ -82,6 +82,7 @@ class Action {
     BackendCompileJobClass,
     FileTableTformJobClass,
     AppendFooterJobClass,
+    SpirvToIrWrapperJobClass,
     StaticLibJobClass,
 
     JobClassFirst = PreprocessJobClass,
@@ -858,6 +859,17 @@ class AppendFooterJobAction : public JobAction {
   }
 };
 
+class SpirvToIrWrapperJobAction : public JobAction {
+  void anchor() override;
+
+public:
+  SpirvToIrWrapperJobAction(Action *Input, types::ID Type);
+
+  static bool classof(const Action *A) {
+    return A->getKind() == SpirvToIrWrapperJobClass;
+  }
+};
+
 class StaticLibJobAction : public JobAction {
   void anchor() override;
 

clang/include/clang/Driver/ToolChain.h

@@ -158,6 +158,7 @@ class ToolChain {
   mutable std::unique_ptr<Tool> BackendCompiler;
   mutable std::unique_ptr<Tool> AppendFooter;
   mutable std::unique_ptr<Tool> FileTableTform;
+  mutable std::unique_ptr<Tool> SpirvToIrWrapper;
 
   Tool *getClang() const;
   Tool *getFlang() const;
@@ -175,6 +176,7 @@ class ToolChain {
   Tool *getBackendCompiler() const;
   Tool *getAppendFooter() const;
   Tool *getTableTform() const;
+  Tool *getSpirvToIrWrapper() const;
 
   mutable bool SanitizerArgsChecked = false;
   mutable std::unique_ptr<XRayArgs> XRayArguments;

clang/lib/Driver/Action.cpp

@@ -61,6 +61,8 @@ const char *Action::getClassName(ActionClass AC) {
     return "static-lib-linker";
   case ForEachWrappingClass:
     return "foreach";
+  case SpirvToIrWrapperJobClass:
+    return "spirv-to-ir-wrapper";
   }
 
   llvm_unreachable("invalid class");
@@ -544,6 +546,12 @@ void StaticLibJobAction::anchor() {}
 StaticLibJobAction::StaticLibJobAction(ActionList &Inputs, types::ID Type)
     : JobAction(StaticLibJobClass, Inputs, Type) {}
 
+void SpirvToIrWrapperJobAction::anchor() {}
+
+SpirvToIrWrapperJobAction::SpirvToIrWrapperJobAction(Action *Input,
+                                                     types::ID Type)
+    : JobAction(SpirvToIrWrapperJobClass, Input, Type) {}
+
 ForEachWrappingAction::ForEachWrappingAction(JobAction *TFormInput,
                                              JobAction *Job)
     : Action(ForEachWrappingClass, {TFormInput, Job}, Job->getType()) {}

clang/lib/Driver/Driver.cpp

@@ -3089,6 +3089,14 @@ bool Driver::checkForOffloadStaticLib(Compilation &C,
   return false;
 }
 
+/// Check whether the given input tree contains any clang-offload-dependency
+/// actions.
+static bool ContainsOffloadDepsAction(const Action *A) {
+  if (isa<OffloadDepsJobAction>(A))
+    return true;
+  return llvm::any_of(A->inputs(), ContainsOffloadDepsAction);
+}
+
 namespace {
 /// Provides a convenient interface for different programming models to generate
 /// the required device actions.
@@ -4614,7 +4622,19 @@ class OffloadingActionBuilder final {
             DA.add(*DeviceWrappingAction, *TC, BoundArch, Action::OFK_SYCL);
             continue;
           } else if (!types::isFPGA(Input->getType())) {
-            LinkObjects.push_back(Input);
+            // No need for any conversion if we are coming in from the
+            // clang-offload-deps or regular compilation path.
+            if (isNVPTX || isAMDGCN || ContainsOffloadDepsAction(Input) ||
+                ContainsCompileOrAssembleAction(Input)) {
+              LinkObjects.push_back(Input);
+              continue;
+            }
+            Action *ConvertSPIRVAction =
+                C.MakeAction<SpirvToIrWrapperJobAction>(
+                    Input, Input->getType() == types::TY_Archive
+                               ? types::TY_Tempfilelist
+                               : types::TY_LLVM_BC);
+            LinkObjects.push_back(ConvertSPIRVAction);
           }
         }
         if (LinkObjects.empty())

clang/lib/Driver/ToolChain.cpp

@@ -371,6 +371,12 @@ Tool *ToolChain::getTableTform() const {
   return FileTableTform.get();
 }
 
+Tool *ToolChain::getSpirvToIrWrapper() const {
+  if (!SpirvToIrWrapper)
+    SpirvToIrWrapper.reset(new tools::SpirvToIrWrapper(*this));
+  return SpirvToIrWrapper.get();
+}
+
 Tool *ToolChain::getTool(Action::ActionClass AC) const {
   switch (AC) {
   case Action::AssembleJobClass:
@@ -431,6 +437,9 @@ Tool *ToolChain::getTool(Action::ActionClass AC) const {
 
   case Action::FileTableTformJobClass:
     return getTableTform();
+
+  case Action::SpirvToIrWrapperJobClass:
+    return getSpirvToIrWrapper();
   }
 
   llvm_unreachable("Invalid tool kind.");

clang/lib/Driver/ToolChains/Clang.cpp

@@ -8590,6 +8590,9 @@ void OffloadBundler::ConstructJobMultipleOutputs(
   if (IsFPGADepUnbundle)
     TypeArg = "o";
 
+  if (InputType == types::TY_Archive && getToolChain().getTriple().isSPIR())
+    TypeArg = "aoo";
+
   // Get the type.
   CmdArgs.push_back(TCArgs.MakeArgString(Twine("-type=") + TypeArg));
 
@@ -9393,3 +9396,38 @@ void AppendFooter::ConstructJob(Compilation &C, const JobAction &JA,
       TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())),
       CmdArgs, None));
 }
+
+void SpirvToIrWrapper::ConstructJob(Compilation &C, const JobAction &JA,
+                                    const InputInfo &Output,
+                                    const InputInfoList &Inputs,
+                                    const llvm::opt::ArgList &TCArgs,
+                                    const char *LinkingOutput) const {
+  InputInfoList ForeachInputs;
+  ArgStringList CmdArgs;
+
+  assert(Inputs.size() == 1 &&
+         "Only one input expected to spirv-to-ir-wrapper");
+
+  // Input File
+  for (const auto &I : Inputs) {
+    if (I.getType() == types::TY_Archive)
+      ForeachInputs.push_back(I);
+    addArgs(CmdArgs, TCArgs, {I.getFilename()});
+  }
+
+  // Output File
+  addArgs(CmdArgs, TCArgs, {"-o", Output.getFilename()});
+
+  auto Cmd = std::make_unique<Command>(
+      JA, *this, ResponseFileSupport::None(),
+      TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())),
+      CmdArgs, None);
+  if (!ForeachInputs.empty()) {
+    StringRef ParallelJobs =
+        TCArgs.getLastArgValue(options::OPT_fsycl_max_parallel_jobs_EQ);
+    tools::SYCL::constructLLVMForeachCommand(
+        C, JA, std::move(Cmd), ForeachInputs, Output, this, "",
+        types::getTypeTempSuffix(types::TY_Tempfilelist), ParallelJobs);
+  } else
+    C.addCommand(std::move(Cmd));
+}

clang/lib/Driver/ToolChains/Clang.h

@@ -266,6 +266,21 @@ class LLVM_LIBRARY_VISIBILITY AppendFooter final : public Tool {
                     const char *LinkingOutput) const override;
 };
 
+/// SPIR-V to LLVM-IR wrapper tool
+class LLVM_LIBRARY_VISIBILITY SpirvToIrWrapper final : public Tool {
+public:
+  SpirvToIrWrapper(const ToolChain &TC)
+      : Tool("Convert SPIR-V to LLVM-IR if needed", "spirv-to-ir-wrapper", TC) {
+  }
+
+  bool hasIntegratedCPP() const override { return false; }
+  bool hasGoodDiagnostics() const override { return true; }
+  void ConstructJob(Compilation &C, const JobAction &JA,
+                    const InputInfo &Output, const InputInfoList &Inputs,
+                    const llvm::opt::ArgList &TCArgs,
+                    const char *LinkingOutput) const override;
+};
+
 } // end namespace tools
 
 } // end namespace driver

clang/lib/Driver/ToolChains/SYCL.cpp

@@ -186,8 +186,8 @@ const char *SYCL::Linker::constructLLVMLinkCommand(
     const ArgList &Args, StringRef SubArchName, StringRef OutputFilePrefix,
     const InputInfoList &InputFiles) const {
   // Split inputs into libraries which have 'archive' type and other inputs
-  // which can be either objects or list files. Objects/list files are linked
-  // together in a usual way, but the libraries need to be linked differently.
+  // which can be either objects or list files. Object files are linked together
+  // in a usual way, but the libraries/list files need to be linked differently.
   // We need to fetch only required symbols from the libraries. With the current
   // llvm-link command line interface that can be achieved with two step
   // linking: at the first step we will link objects into an intermediate
@@ -236,7 +236,7 @@ const char *SYCL::Linker::constructLLVMLinkCommand(
       if (II.getType() == types::TY_Tempfilelist) {
         // Pass the unbundled list with '@' to be processed.
         std::string FileName(II.getFilename());
-        Objs.push_back(C.getArgs().MakeArgString("@" + FileName));
+        Libs.push_back(C.getArgs().MakeArgString("@" + FileName));
       } else if (II.getType() == types::TY_Archive && !LinkSYCLDeviceLibs) {
         Libs.push_back(II.getFilename());
       } else

clang/test/Driver/sycl-device-lib.cpp

@@ -139,5 +139,5 @@
 // RUN: touch libsycl-crt.o
 // RUN: %clangxx -fsycl libsycl-crt.o --sysroot=%S/Inputs/SYCL -### 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=SYCL_LLVM_LINK_USER_ONLY_NEEDED
-// SYCL_LLVM_LINK_USER_ONLY_NEEDED: llvm-link{{.*}}  "{{.*}}.o" "-o" "{{.*}}.bc" "--suppress-warnings"
+// SYCL_LLVM_LINK_USER_ONLY_NEEDED: llvm-link{{.*}}  "{{.*}}.bc" "-o" "{{.*}}.bc" "--suppress-warnings"
 // SYCL_LLVM_LINK_USER_ONLY_NEEDED: llvm-link{{.*}}  "-only-needed" "{{.*}}" "-o" "{{.*}}.bc" "--suppress-warnings"

clang/test/Driver/sycl-intelfpga-aoco.cpp

@@ -83,24 +83,26 @@
 // CHK-FPGA-AOCO-PHASES-EMU: 13: clang-offload-deps, {12}, ir, (host-sycl)
 // CHK-FPGA-AOCO-PHASES-EMU: 14: input, "[[INPUTA]]", archive
 // CHK-FPGA-AOCO-PHASES-EMU: 15: clang-offload-unbundler, {14}, archive
-// CHK-FPGA-AOCO-PHASES-EMU: 16: linker, {6, 13, 15}, ir, (device-sycl)
-// CHK-FPGA-AOCO-PHASES-EMU: 17: sycl-post-link, {16}, tempfiletable, (device-sycl)
-// CHK-FPGA-AOCO-PHASES-EMU: 18: file-table-tform, {17}, tempfilelist, (device-sycl)
-// CHK-FPGA-AOCO-PHASES-EMU: 19: llvm-spirv, {18}, tempfilelist, (device-sycl)
-// CHK-FPGA-AOCO-PHASES-EMU: 20: input, "[[INPUTA]]", archive
-// CHK-FPGA-AOCO-PHASES-EMU: 21: clang-offload-unbundler, {20}, fpga_dep_list
-// CHK-FPGA-AOCO-PHASES-EMU: 22: backend-compiler, {19, 21}, fpga_aocx, (device-sycl)
-// CHK-FPGA-AOCO-PHASES-EMU: 23: file-table-tform, {17, 22}, tempfiletable, (device-sycl)
-// CHK-FPGA-AOCO-PHASES-EMU: 24: clang-offload-wrapper, {23}, object, (device-sycl)
-// CHK-FPGA-AOCO-PHASES-EMU: 25: offload, "host-sycl (x86_64-unknown-linux-gnu)" {11}, "device-sycl (spir64_fpga-unknown-unknown)" {24}, image
+// CHK-FPGA-AOCO-PHASES-EMU: 16: spirv-to-ir-wrapper, {15}, tempfilelist, (device-sycl)
+// CHK-FPGA-AOCO-PHASES-EMU: 17: linker, {6, 13, 16}, ir, (device-sycl)
+// CHK-FPGA-AOCO-PHASES-EMU: 18: sycl-post-link, {17}, tempfiletable, (device-sycl)
+// CHK-FPGA-AOCO-PHASES-EMU: 19: file-table-tform, {18}, tempfilelist, (device-sycl)
+// CHK-FPGA-AOCO-PHASES-EMU: 20: llvm-spirv, {19}, tempfilelist, (device-sycl)
+// CHK-FPGA-AOCO-PHASES-EMU: 21: input, "[[INPUTA]]", archive
+// CHK-FPGA-AOCO-PHASES-EMU: 22: clang-offload-unbundler, {21}, fpga_dep_list
+// CHK-FPGA-AOCO-PHASES-EMU: 23: backend-compiler, {20, 22}, fpga_aocx, (device-sycl)
+// CHK-FPGA-AOCO-PHASES-EMU: 24: file-table-tform, {18, 23}, tempfiletable, (device-sycl)
+// CHK-FPGA-AOCO-PHASES-EMU: 25: clang-offload-wrapper, {24}, object, (device-sycl)
+// CHK-FPGA-AOCO-PHASES-EMU: 26: offload, "host-sycl (x86_64-unknown-linux-gnu)" {11}, "device-sycl (spir64_fpga-unknown-unknown)" {25}, image
 
 /// aoco emulation test, checking tools
 // RUN:  %clangxx -target x86_64-unknown-linux-gnu -fsycl -fno-sycl-device-lib=all -fintelfpga %t_aoco.a -### %s 2>&1 \
 // RUN:  | FileCheck -check-prefixes=CHK-FPGA-AOCO-EMU,CHK-FPGA-AOCO-EMU-LIN %s
 // RUN:  %clang_cl -fsycl -fno-sycl-device-lib=all -fintelfpga %t_aoco_cl.a -### %s 2>&1 \
 // RUN:  | FileCheck -check-prefixes=CHK-FPGA-AOCO-EMU,CHK-FPGA-AOCO-EMU-WIN %s
-// CHK-FPGA-AOCO-EMU: clang-offload-bundler{{.*}} "-type=a" "-targets=sycl-spir64_fpga-unknown-unknown" "-inputs=[[INPUTLIB:.+\.a]]" "-outputs=[[OUTLIB:.+\.a]]" "-unbundle"
-// CHK-FPGA-AOCO-EMU: llvm-link{{.*}} "[[OUTLIB]]" "-o" "[[LINKEDBC:.+\.bc]]"
+// CHK-FPGA-AOCO-EMU: clang-offload-bundler{{.*}} "-type=aoo" "-targets=sycl-spir64_fpga-unknown-unknown" "-inputs=[[INPUTLIB:.+\.a]]" "-outputs=[[OUTLIB:.+\.a]]" "-unbundle"
+// CHK-FPGA-AOCO-EMU: llvm-foreach{{.*}} "--out-ext=txt" "--in-file-list=[[OUTLIB]]" "--in-replace=[[OUTLIB]]" "--out-file-list=[[DEVICELIST:.+\.txt]]" "--out-replace=[[DEVICELIST]]" "--" {{.*}}spirv-to-ir-wrapper{{.*}} "[[OUTLIB]]" "-o" "[[DEVICELIST]]"
+// CHK-FPGA-AOCO-EMU: llvm-link{{.*}} "@[[DEVICELIST]]" "-o" "[[LINKEDBC:.+\.bc]]"
 // CHK-FPGA-AOCO-EMU: sycl-post-link{{.*}} "-split-esimd"{{.*}} "-O2" "-spec-const=default" "-o" "[[SPLTABLE:.+\.table]]" "[[LINKEDBC]]"
 // CHK-FPGA-AOCO-EMU: file-table-tform{{.*}} "-o" "[[TABLEOUT:.+\.txt]]" "[[SPLTABLE]]"
 // CHK-FPGA-AOCO-EMU: llvm-spirv{{.*}} "-o" "[[TARGSPV:.+\.txt]]" {{.*}} "[[TABLEOUT]]"