diff --git a/llvm/include/llvm/Transforms/Coroutines/ABI.h b/llvm/include/llvm/Transforms/Coroutines/ABI.h new file mode 100644 index 0000000000000..b36495326f213 --- /dev/null +++ b/llvm/include/llvm/Transforms/Coroutines/ABI.h @@ -0,0 +1,115 @@ +//===- ABI.h - Coroutine ABI Transformers ---------------------*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// This file declares the pass that analyzes a function for coroutine intrs and +// a transformer class that contains methods for handling different steps of +// coroutine lowering. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_COROUTINES_ABI_H +#define LLVM_TRANSFORMS_COROUTINES_ABI_H + +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Transforms/Coroutines/CoroShape.h" +#include "llvm/Transforms/Coroutines/MaterializationUtils.h" +#include "llvm/Transforms/Coroutines/SuspendCrossingInfo.h" + +namespace llvm { + +class Function; + +namespace coro { + +// This interface/API is to provide an object oriented way to implement ABI +// functionality. This is intended to replace use of switch(ABI) to perform +// ABI specific behavior. The ABIs (e.g. Switch, Async, Retcon{Once}) are the +// common ABIs. However, specific users often need to modify the behavior of +// these, such as for C++20 or Swift. This can be accomplished by inheriting +// one of the common ABIs and overriding one or more of the methods to create +// a custom ABI. The custom ABI is specified with the coro.begin.custom.abi +// intrinsic instead of the coro.begin intrinsic by providing an i32 in the +// last argument. This is used to lookup a generator for the custom ABI from +// a set of generators provided to the CoroSplitPass constructor. +// +class BaseABI { +public: + BaseABI(Function &F, Shape &S) + : F(F), Shape(S), IsMaterializable(coro::isTriviallyMaterializable) {} + + BaseABI(Function &F, coro::Shape &S, + std::function IsMaterializable) + : F(F), Shape(S), IsMaterializable(IsMaterializable) {} + + // Initialize the coroutine ABI + virtual void init() = 0; + + // Allocate the coroutine frame and do spill/reload as needed. + virtual void buildCoroutineFrame(bool OptimizeFrame); + + // Perform the function splitting according to the ABI. + virtual void splitCoroutine(Function &F, coro::Shape &Shape, + SmallVectorImpl &Clones, + TargetTransformInfo &TTI) = 0; + + Function &F; + coro::Shape &Shape; + + // Callback used by coro::BaseABI::buildCoroutineFrame for rematerialization. + // It is provided to coro::doMaterializations(..). + std::function IsMaterializable; +}; + +class SwitchABI : public BaseABI { +public: + SwitchABI(Function &F, coro::Shape &S) : BaseABI(F, S) {} + + SwitchABI(Function &F, coro::Shape &S, + std::function IsMaterializable) + : BaseABI(F, S, IsMaterializable) {} + + void init() override; + + void splitCoroutine(Function &F, coro::Shape &Shape, + SmallVectorImpl &Clones, + TargetTransformInfo &TTI) override; +}; + +class AsyncABI : public BaseABI { +public: + AsyncABI(Function &F, coro::Shape &S) : BaseABI(F, S) {} + + AsyncABI(Function &F, coro::Shape &S, + std::function IsMaterializable) + : BaseABI(F, S, IsMaterializable) {} + + void init() override; + + void splitCoroutine(Function &F, coro::Shape &Shape, + SmallVectorImpl &Clones, + TargetTransformInfo &TTI) override; +}; + +class AnyRetconABI : public BaseABI { +public: + AnyRetconABI(Function &F, coro::Shape &S) : BaseABI(F, S) {} + + AnyRetconABI(Function &F, coro::Shape &S, + std::function IsMaterializable) + : BaseABI(F, S, IsMaterializable) {} + + void init() override; + + void splitCoroutine(Function &F, coro::Shape &Shape, + SmallVectorImpl &Clones, + TargetTransformInfo &TTI) override; +}; + +} // end namespace coro + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_COROUTINES_ABI_H diff --git a/llvm/lib/Transforms/Coroutines/CoroInstr.h b/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h similarity index 90% rename from llvm/lib/Transforms/Coroutines/CoroInstr.h rename to llvm/include/llvm/Transforms/Coroutines/CoroInstr.h index a31703fe01304..4e63cbd08a938 100644 --- a/llvm/lib/Transforms/Coroutines/CoroInstr.h +++ b/llvm/include/llvm/Transforms/Coroutines/CoroInstr.h @@ -22,8 +22,8 @@ // the Coroutine library. //===----------------------------------------------------------------------===// -#ifndef LLVM_LIB_TRANSFORMS_COROUTINES_COROINSTR_H -#define LLVM_LIB_TRANSFORMS_COROUTINES_COROINSTR_H +#ifndef LLVM_TRANSFORMS_COROUTINES_COROINSTR_H +#define LLVM_TRANSFORMS_COROUTINES_COROINSTR_H #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IntrinsicInst.h" @@ -32,7 +32,7 @@ namespace llvm { /// This class represents the llvm.coro.subfn.addr instruction. -class LLVM_LIBRARY_VISIBILITY CoroSubFnInst : public IntrinsicInst { +class CoroSubFnInst : public IntrinsicInst { enum { FrameArg, IndexArg }; public: @@ -67,7 +67,7 @@ class LLVM_LIBRARY_VISIBILITY CoroSubFnInst : public IntrinsicInst { }; /// This represents the llvm.coro.alloc instruction. -class LLVM_LIBRARY_VISIBILITY CoroAllocInst : public IntrinsicInst { +class CoroAllocInst : public IntrinsicInst { public: // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -82,7 +82,7 @@ class LLVM_LIBRARY_VISIBILITY CoroAllocInst : public IntrinsicInst { // FIXME: add callback metadata // FIXME: make a proper IntrinisicInst. Currently this is not possible, // because llvm.coro.await.suspend.* can be invoked. -class LLVM_LIBRARY_VISIBILITY CoroAwaitSuspendInst : public CallBase { +class CoroAwaitSuspendInst : public CallBase { enum { AwaiterArg, FrameArg, WrapperArg }; public: @@ -112,7 +112,7 @@ class LLVM_LIBRARY_VISIBILITY CoroAwaitSuspendInst : public CallBase { }; /// This represents a common base class for llvm.coro.id instructions. -class LLVM_LIBRARY_VISIBILITY AnyCoroIdInst : public IntrinsicInst { +class AnyCoroIdInst : public IntrinsicInst { public: CoroAllocInst *getCoroAlloc() { for (User *U : users()) @@ -143,7 +143,7 @@ class LLVM_LIBRARY_VISIBILITY AnyCoroIdInst : public IntrinsicInst { }; /// This represents the llvm.coro.id instruction. -class LLVM_LIBRARY_VISIBILITY CoroIdInst : public AnyCoroIdInst { +class CoroIdInst : public AnyCoroIdInst { enum { AlignArg, PromiseArg, CoroutineArg, InfoArg }; public: @@ -232,7 +232,7 @@ class LLVM_LIBRARY_VISIBILITY CoroIdInst : public AnyCoroIdInst { /// This represents either the llvm.coro.id.retcon or /// llvm.coro.id.retcon.once instruction. -class LLVM_LIBRARY_VISIBILITY AnyCoroIdRetconInst : public AnyCoroIdInst { +class AnyCoroIdRetconInst : public AnyCoroIdInst { enum { SizeArg, AlignArg, StorageArg, PrototypeArg, AllocArg, DeallocArg }; public: @@ -246,9 +246,7 @@ class LLVM_LIBRARY_VISIBILITY AnyCoroIdRetconInst : public AnyCoroIdInst { return cast(getArgOperand(AlignArg))->getAlignValue(); } - Value *getStorage() const { - return getArgOperand(StorageArg); - } + Value *getStorage() const { return getArgOperand(StorageArg); } /// Return the prototype for the continuation function. The type, /// attributes, and calling convention of the continuation function(s) @@ -270,8 +268,8 @@ class LLVM_LIBRARY_VISIBILITY AnyCoroIdRetconInst : public AnyCoroIdInst { // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { auto ID = I->getIntrinsicID(); - return ID == Intrinsic::coro_id_retcon - || ID == Intrinsic::coro_id_retcon_once; + return ID == Intrinsic::coro_id_retcon || + ID == Intrinsic::coro_id_retcon_once; } static bool classof(const Value *V) { return isa(V) && classof(cast(V)); @@ -279,8 +277,7 @@ class LLVM_LIBRARY_VISIBILITY AnyCoroIdRetconInst : public AnyCoroIdInst { }; /// This represents the llvm.coro.id.retcon instruction. -class LLVM_LIBRARY_VISIBILITY CoroIdRetconInst - : public AnyCoroIdRetconInst { +class CoroIdRetconInst : public AnyCoroIdRetconInst { public: // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -292,8 +289,7 @@ class LLVM_LIBRARY_VISIBILITY CoroIdRetconInst }; /// This represents the llvm.coro.id.retcon.once instruction. -class LLVM_LIBRARY_VISIBILITY CoroIdRetconOnceInst - : public AnyCoroIdRetconInst { +class CoroIdRetconOnceInst : public AnyCoroIdRetconInst { public: // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -305,7 +301,7 @@ class LLVM_LIBRARY_VISIBILITY CoroIdRetconOnceInst }; /// This represents the llvm.coro.id.async instruction. -class LLVM_LIBRARY_VISIBILITY CoroIdAsyncInst : public AnyCoroIdInst { +class CoroIdAsyncInst : public AnyCoroIdInst { enum { SizeArg, AlignArg, StorageArg, AsyncFuncPtrArg }; public: @@ -356,7 +352,7 @@ class LLVM_LIBRARY_VISIBILITY CoroIdAsyncInst : public AnyCoroIdInst { }; /// This represents the llvm.coro.context.alloc instruction. -class LLVM_LIBRARY_VISIBILITY CoroAsyncContextAllocInst : public IntrinsicInst { +class CoroAsyncContextAllocInst : public IntrinsicInst { enum { AsyncFuncPtrArg }; public: @@ -375,8 +371,7 @@ class LLVM_LIBRARY_VISIBILITY CoroAsyncContextAllocInst : public IntrinsicInst { }; /// This represents the llvm.coro.context.dealloc instruction. -class LLVM_LIBRARY_VISIBILITY CoroAsyncContextDeallocInst - : public IntrinsicInst { +class CoroAsyncContextDeallocInst : public IntrinsicInst { enum { AsyncContextArg }; public: @@ -396,7 +391,7 @@ class LLVM_LIBRARY_VISIBILITY CoroAsyncContextDeallocInst /// This represents the llvm.coro.async.resume instruction. /// During lowering this is replaced by the resume function of a suspend point /// (the continuation function). -class LLVM_LIBRARY_VISIBILITY CoroAsyncResumeInst : public IntrinsicInst { +class CoroAsyncResumeInst : public IntrinsicInst { public: // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -408,7 +403,7 @@ class LLVM_LIBRARY_VISIBILITY CoroAsyncResumeInst : public IntrinsicInst { }; /// This represents the llvm.coro.async.size.replace instruction. -class LLVM_LIBRARY_VISIBILITY CoroAsyncSizeReplace : public IntrinsicInst { +class CoroAsyncSizeReplace : public IntrinsicInst { public: // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -420,7 +415,7 @@ class LLVM_LIBRARY_VISIBILITY CoroAsyncSizeReplace : public IntrinsicInst { }; /// This represents the llvm.coro.frame instruction. -class LLVM_LIBRARY_VISIBILITY CoroFrameInst : public IntrinsicInst { +class CoroFrameInst : public IntrinsicInst { public: // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -432,7 +427,7 @@ class LLVM_LIBRARY_VISIBILITY CoroFrameInst : public IntrinsicInst { }; /// This represents the llvm.coro.free instruction. -class LLVM_LIBRARY_VISIBILITY CoroFreeInst : public IntrinsicInst { +class CoroFreeInst : public IntrinsicInst { enum { IdArg, FrameArg }; public: @@ -447,8 +442,8 @@ class LLVM_LIBRARY_VISIBILITY CoroFreeInst : public IntrinsicInst { } }; -/// This class represents the llvm.coro.begin instruction. -class LLVM_LIBRARY_VISIBILITY CoroBeginInst : public IntrinsicInst { +/// This class represents the llvm.coro.begin instructions. +class CoroBeginInst : public IntrinsicInst { enum { IdArg, MemArg }; public: @@ -468,7 +463,7 @@ class LLVM_LIBRARY_VISIBILITY CoroBeginInst : public IntrinsicInst { }; /// This represents the llvm.coro.save instruction. -class LLVM_LIBRARY_VISIBILITY CoroSaveInst : public IntrinsicInst { +class CoroSaveInst : public IntrinsicInst { public: // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -480,7 +475,7 @@ class LLVM_LIBRARY_VISIBILITY CoroSaveInst : public IntrinsicInst { }; /// This represents the llvm.coro.promise instruction. -class LLVM_LIBRARY_VISIBILITY CoroPromiseInst : public IntrinsicInst { +class CoroPromiseInst : public IntrinsicInst { enum { FrameArg, AlignArg, FromArg }; public: @@ -505,7 +500,7 @@ class LLVM_LIBRARY_VISIBILITY CoroPromiseInst : public IntrinsicInst { } }; -class LLVM_LIBRARY_VISIBILITY AnyCoroSuspendInst : public IntrinsicInst { +class AnyCoroSuspendInst : public IntrinsicInst { public: CoroSaveInst *getCoroSave() const; @@ -521,7 +516,7 @@ class LLVM_LIBRARY_VISIBILITY AnyCoroSuspendInst : public IntrinsicInst { }; /// This represents the llvm.coro.suspend instruction. -class LLVM_LIBRARY_VISIBILITY CoroSuspendInst : public AnyCoroSuspendInst { +class CoroSuspendInst : public AnyCoroSuspendInst { enum { SaveArg, FinalArg }; public: @@ -553,7 +548,7 @@ inline CoroSaveInst *AnyCoroSuspendInst::getCoroSave() const { } /// This represents the llvm.coro.suspend.async instruction. -class LLVM_LIBRARY_VISIBILITY CoroSuspendAsyncInst : public AnyCoroSuspendInst { +class CoroSuspendAsyncInst : public AnyCoroSuspendInst { public: enum { StorageArgNoArg, @@ -594,7 +589,7 @@ class LLVM_LIBRARY_VISIBILITY CoroSuspendAsyncInst : public AnyCoroSuspendInst { }; /// This represents the llvm.coro.suspend.retcon instruction. -class LLVM_LIBRARY_VISIBILITY CoroSuspendRetconInst : public AnyCoroSuspendInst { +class CoroSuspendRetconInst : public AnyCoroSuspendInst { public: op_iterator value_begin() { return arg_begin(); } const_op_iterator value_begin() const { return arg_begin(); } @@ -619,7 +614,7 @@ class LLVM_LIBRARY_VISIBILITY CoroSuspendRetconInst : public AnyCoroSuspendInst }; /// This represents the llvm.coro.size instruction. -class LLVM_LIBRARY_VISIBILITY CoroSizeInst : public IntrinsicInst { +class CoroSizeInst : public IntrinsicInst { public: // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -631,7 +626,7 @@ class LLVM_LIBRARY_VISIBILITY CoroSizeInst : public IntrinsicInst { }; /// This represents the llvm.coro.align instruction. -class LLVM_LIBRARY_VISIBILITY CoroAlignInst : public IntrinsicInst { +class CoroAlignInst : public IntrinsicInst { public: // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -643,7 +638,7 @@ class LLVM_LIBRARY_VISIBILITY CoroAlignInst : public IntrinsicInst { }; /// This represents the llvm.end.results instruction. -class LLVM_LIBRARY_VISIBILITY CoroEndResults : public IntrinsicInst { +class CoroEndResults : public IntrinsicInst { public: op_iterator retval_begin() { return arg_begin(); } const_op_iterator retval_begin() const { return arg_begin(); } @@ -671,7 +666,7 @@ class LLVM_LIBRARY_VISIBILITY CoroEndResults : public IntrinsicInst { } }; -class LLVM_LIBRARY_VISIBILITY AnyCoroEndInst : public IntrinsicInst { +class AnyCoroEndInst : public IntrinsicInst { enum { FrameArg, UnwindArg, TokenArg }; public: @@ -700,7 +695,7 @@ class LLVM_LIBRARY_VISIBILITY AnyCoroEndInst : public IntrinsicInst { }; /// This represents the llvm.coro.end instruction. -class LLVM_LIBRARY_VISIBILITY CoroEndInst : public AnyCoroEndInst { +class CoroEndInst : public AnyCoroEndInst { public: // Methods to support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -712,7 +707,7 @@ class LLVM_LIBRARY_VISIBILITY CoroEndInst : public AnyCoroEndInst { }; /// This represents the llvm.coro.end instruction. -class LLVM_LIBRARY_VISIBILITY CoroAsyncEndInst : public AnyCoroEndInst { +class CoroAsyncEndInst : public AnyCoroEndInst { enum { FrameArg, UnwindArg, MustTailCallFuncArg }; public: @@ -736,12 +731,11 @@ class LLVM_LIBRARY_VISIBILITY CoroAsyncEndInst : public AnyCoroEndInst { }; /// This represents the llvm.coro.alloca.alloc instruction. -class LLVM_LIBRARY_VISIBILITY CoroAllocaAllocInst : public IntrinsicInst { +class CoroAllocaAllocInst : public IntrinsicInst { enum { SizeArg, AlignArg }; + public: - Value *getSize() const { - return getArgOperand(SizeArg); - } + Value *getSize() const { return getArgOperand(SizeArg); } Align getAlignment() const { return cast(getArgOperand(AlignArg))->getAlignValue(); } @@ -756,8 +750,9 @@ class LLVM_LIBRARY_VISIBILITY CoroAllocaAllocInst : public IntrinsicInst { }; /// This represents the llvm.coro.alloca.get instruction. -class LLVM_LIBRARY_VISIBILITY CoroAllocaGetInst : public IntrinsicInst { +class CoroAllocaGetInst : public IntrinsicInst { enum { AllocArg }; + public: CoroAllocaAllocInst *getAlloc() const { return cast(getArgOperand(AllocArg)); @@ -773,8 +768,9 @@ class LLVM_LIBRARY_VISIBILITY CoroAllocaGetInst : public IntrinsicInst { }; /// This represents the llvm.coro.alloca.free instruction. -class LLVM_LIBRARY_VISIBILITY CoroAllocaFreeInst : public IntrinsicInst { +class CoroAllocaFreeInst : public IntrinsicInst { enum { AllocArg }; + public: CoroAllocaAllocInst *getAlloc() const { return cast(getArgOperand(AllocArg)); @@ -789,6 +785,17 @@ class LLVM_LIBRARY_VISIBILITY CoroAllocaFreeInst : public IntrinsicInst { } }; +namespace coro { + +// Check for structural coroutine intrinsics that should not be spilled into +// the coroutine frame. +bool isCoroutineStructureIntrinsic(Instruction &I); + +// Check if BasicBlock suspends the coroutine +bool isSuspendBlock(BasicBlock *BB); + +} // End namespace coro. + } // End namespace llvm. -#endif +#endif // LLVM_TRANSFORMS_COROUTINES_COROINSTR_H diff --git a/llvm/lib/Transforms/Coroutines/CoroShape.h b/llvm/include/llvm/Transforms/Coroutines/CoroShape.h similarity index 82% rename from llvm/lib/Transforms/Coroutines/CoroShape.h rename to llvm/include/llvm/Transforms/Coroutines/CoroShape.h index f5798b63bf732..f0abc05a6dff7 100644 --- a/llvm/lib/Transforms/Coroutines/CoroShape.h +++ b/llvm/include/llvm/Transforms/Coroutines/CoroShape.h @@ -12,9 +12,9 @@ #ifndef LLVM_TRANSFORMS_COROUTINES_COROSHAPE_H #define LLVM_TRANSFORMS_COROUTINES_COROSHAPE_H -#include "CoroInstr.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Coroutines/CoroInstr.h" namespace llvm { @@ -22,6 +22,10 @@ class CallGraph; namespace coro { +// This enum is used by CoroSplitPass's internal methods for implementing ABI +// specific behavior. Ideally, this would not be necessary and all behavior +// would be implemented by the ABI objects. However, it is not practical and +// not necessary to move such code to the ABI objects. enum class ABI { /// The "resume-switch" lowering, where there are separate resume and /// destroy functions that are shared between all suspend points. The @@ -49,16 +53,50 @@ enum class ABI { // Holds structural Coroutine Intrinsics for a particular function and other // values used during CoroSplit pass. -struct LLVM_LIBRARY_VISIBILITY Shape { - CoroBeginInst *CoroBegin; +struct Shape { + CoroBeginInst *CoroBegin = nullptr; SmallVector CoroEnds; SmallVector CoroSizes; SmallVector CoroAligns; SmallVector CoroSuspends; - SmallVector SwiftErrorOps; SmallVector CoroAwaitSuspends; SmallVector SymmetricTransfers; + // Values invalidated by invalidateCoroutine() and tidyCoroutine() + SmallVector CoroFrames; + SmallVector UnusedCoroSaves; + + // Values invalidated by replaceSwiftErrorOps() + SmallVector SwiftErrorOps; + + void clear() { + CoroBegin = nullptr; + CoroEnds.clear(); + CoroSizes.clear(); + CoroAligns.clear(); + CoroSuspends.clear(); + CoroAwaitSuspends.clear(); + SymmetricTransfers.clear(); + + CoroFrames.clear(); + UnusedCoroSaves.clear(); + + SwiftErrorOps.clear(); + + FrameTy = nullptr; + FramePtr = nullptr; + AllocaSpillBlock = nullptr; + } + + // Scan the function and collect the above intrinsics for later processing + void analyze(Function &F); + // If for some reason, we were not able to find coro.begin, bailout. + void invalidateCoroutine(Function &F); + // Perform ABI related initial transformation + void initABI(); + // Remove orphaned and unnecessary intrinsics + void tidyCoroutine(); + // Field indexes for special fields in the switch lowering. struct SwitchFieldIndex { enum { @@ -76,14 +114,13 @@ struct LLVM_LIBRARY_VISIBILITY Shape { coro::ABI ABI; - StructType *FrameTy; + // Values below are used by ABIs for lowering. Ideally, these would live in + // the ABI objects along with the helpers that use them. + StructType *FrameTy = nullptr; Align FrameAlign; - uint64_t FrameSize; - Value *FramePtr; - BasicBlock *AllocaSpillBlock; - - /// This would only be true if optimization are enabled. - bool OptimizeFrame; + uint64_t FrameSize = 0; + Value *FramePtr = nullptr; + BasicBlock *AllocaSpillBlock = nullptr; struct SwitchLoweringStorage { SwitchInst *ResumeSwitch; @@ -235,11 +272,7 @@ struct LLVM_LIBRARY_VISIBILITY Shape { void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const; Shape() = default; - explicit Shape(Function &F, bool OptimizeFrame = false) - : OptimizeFrame(OptimizeFrame) { - buildFrom(F); - } - void buildFrom(Function &F); + explicit Shape(Function &F) { analyze(F); } }; } // end namespace coro diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h index a2be1099ff68f..7a0e61c36efcc 100644 --- a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h +++ b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h @@ -18,22 +18,30 @@ #include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Coroutines/ABI.h" namespace llvm { +namespace coro { +class BaseABI; +class Shape; +} // namespace coro + struct CoroSplitPass : PassInfoMixin { - const std::function MaterializableCallback; + // BaseABITy generates an instance of a coro ABI. + using BaseABITy = std::function; CoroSplitPass(bool OptimizeFrame = false); CoroSplitPass(std::function MaterializableCallback, - bool OptimizeFrame = false) - : MaterializableCallback(MaterializableCallback), - OptimizeFrame(OptimizeFrame) {} + bool OptimizeFrame = false); PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR); static bool isRequired() { return true; } + // Generator for an ABI transformer + BaseABITy GenABI; + // Would be true if the Optimization level isn't O0. bool OptimizeFrame; }; diff --git a/llvm/lib/Transforms/Coroutines/MaterializationUtils.h b/llvm/include/llvm/Transforms/Coroutines/MaterializationUtils.h similarity index 76% rename from llvm/lib/Transforms/Coroutines/MaterializationUtils.h rename to llvm/include/llvm/Transforms/Coroutines/MaterializationUtils.h index f391851c97b3b..d8fc0c86a6fb5 100644 --- a/llvm/lib/Transforms/Coroutines/MaterializationUtils.h +++ b/llvm/include/llvm/Transforms/Coroutines/MaterializationUtils.h @@ -6,11 +6,10 @@ // //===----------------------------------------------------------------------===// -#include "SuspendCrossingInfo.h" -#include "llvm/IR/Instruction.h" +#include "llvm/Transforms/Coroutines/SuspendCrossingInfo.h" -#ifndef LIB_TRANSFORMS_COROUTINES_MATERIALIZATIONUTILS_H -#define LIB_TRANSFORMS_COROUTINES_MATERIALIZATIONUTILS_H +#ifndef LLVM_TRANSFORMS_COROUTINES_MATERIALIZATIONUTILS_H +#define LLVM_TRANSFORMS_COROUTINES_MATERIALIZATIONUTILS_H namespace llvm { @@ -27,4 +26,4 @@ void doRematerializations(Function &F, SuspendCrossingInfo &Checker, } // namespace llvm -#endif // LIB_TRANSFORMS_COROUTINES_MATERIALIZATIONUTILS_H +#endif // LLVM_TRANSFORMS_COROUTINES_MATERIALIZATIONUTILS_H diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.h b/llvm/include/llvm/Transforms/Coroutines/SpillUtils.h similarity index 93% rename from llvm/lib/Transforms/Coroutines/SpillUtils.h rename to llvm/include/llvm/Transforms/Coroutines/SpillUtils.h index 8843b611e0842..6cdf83c0603f4 100644 --- a/llvm/lib/Transforms/Coroutines/SpillUtils.h +++ b/llvm/include/llvm/Transforms/Coroutines/SpillUtils.h @@ -6,8 +6,9 @@ // //===----------------------------------------------------------------------===// -#include "CoroInternal.h" -#include "SuspendCrossingInfo.h" +#include "llvm/IR/Dominators.h" +#include "llvm/Transforms/Coroutines/CoroShape.h" +#include "llvm/Transforms/Coroutines/SuspendCrossingInfo.h" #ifndef LLVM_TRANSFORMS_COROUTINES_SPILLINGINFO_H #define LLVM_TRANSFORMS_COROUTINES_SPILLINGINFO_H diff --git a/llvm/lib/Transforms/Coroutines/SuspendCrossingInfo.h b/llvm/include/llvm/Transforms/Coroutines/SuspendCrossingInfo.h similarity index 97% rename from llvm/lib/Transforms/Coroutines/SuspendCrossingInfo.h rename to llvm/include/llvm/Transforms/Coroutines/SuspendCrossingInfo.h index db889966dcf1d..49cae6dde47e5 100644 --- a/llvm/lib/Transforms/Coroutines/SuspendCrossingInfo.h +++ b/llvm/include/llvm/Transforms/Coroutines/SuspendCrossingInfo.h @@ -12,16 +12,16 @@ // ptrs in the BlockToIndexMapping. //===----------------------------------------------------------------------===// -#ifndef LLVM_LIB_TRANSFORMS_COROUTINES_SUSPENDCROSSINGINFO_H -#define LLVM_LIB_TRANSFORMS_COROUTINES_SUSPENDCROSSINGINFO_H +#ifndef LLVM_TRANSFORMS_COROUTINES_SUSPENDCROSSINGINFO_H +#define LLVM_TRANSFORMS_COROUTINES_SUSPENDCROSSINGINFO_H -#include "CoroInstr.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instruction.h" +#include "llvm/Transforms/Coroutines/CoroInstr.h" namespace llvm { diff --git a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp index 5f8efd1a8f32e..f36ae2274c79f 100644 --- a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp @@ -8,12 +8,12 @@ #include "llvm/Transforms/Coroutines/CoroEarly.h" #include "CoroInternal.h" -#include "CoroShape.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Module.h" +#include "llvm/Transforms/Coroutines/CoroShape.h" using namespace llvm; diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp index c08f56b024dfc..a83c9e2638b48 100644 --- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp @@ -16,9 +16,6 @@ //===----------------------------------------------------------------------===// #include "CoroInternal.h" -#include "MaterializationUtils.h" -#include "SpillUtils.h" -#include "SuspendCrossingInfo.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallString.h" @@ -32,6 +29,11 @@ #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/OptimizedStructLayout.h" +#include "llvm/Transforms/Coroutines/ABI.h" +#include "llvm/Transforms/Coroutines/CoroInstr.h" +#include "llvm/Transforms/Coroutines/MaterializationUtils.h" +#include "llvm/Transforms/Coroutines/SpillUtils.h" +#include "llvm/Transforms/Coroutines/SuspendCrossingInfo.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/PromoteMemToReg.h" @@ -233,7 +235,7 @@ class FrameTypeBuilder { /// Side Effects: Because We sort the allocas, the order of allocas in the /// frame may be different with the order in the source code. void addFieldForAllocas(const Function &F, FrameDataInfo &FrameData, - coro::Shape &Shape); + coro::Shape &Shape, bool OptimizeFrame); /// Add a field to this structure. [[nodiscard]] FieldIDType addField(Type *Ty, MaybeAlign MaybeFieldAlignment, @@ -335,7 +337,8 @@ void FrameDataInfo::updateLayoutIndex(FrameTypeBuilder &B) { void FrameTypeBuilder::addFieldForAllocas(const Function &F, FrameDataInfo &FrameData, - coro::Shape &Shape) { + coro::Shape &Shape, + bool OptimizeFrame) { using AllocaSetType = SmallVector; SmallVector NonOverlapedAllocas; @@ -349,7 +352,7 @@ void FrameTypeBuilder::addFieldForAllocas(const Function &F, } }); - if (!Shape.OptimizeFrame) { + if (!OptimizeFrame) { for (const auto &A : FrameData.Allocas) { AllocaInst *Alloca = A.Alloca; NonOverlapedAllocas.emplace_back(AllocaSetType(1, Alloca)); @@ -859,7 +862,8 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape, // ... spills ... // }; static StructType *buildFrameType(Function &F, coro::Shape &Shape, - FrameDataInfo &FrameData) { + FrameDataInfo &FrameData, + bool OptimizeFrame) { LLVMContext &C = F.getContext(); const DataLayout &DL = F.getDataLayout(); StructType *FrameTy = [&] { @@ -904,7 +908,7 @@ static StructType *buildFrameType(Function &F, coro::Shape &Shape, // Because multiple allocas may own the same field slot, // we add allocas to field here. - B.addFieldForAllocas(F, FrameData, Shape); + B.addFieldForAllocas(F, FrameData, Shape, OptimizeFrame); // Add PromiseAlloca to Allocas list so that // 1. updateLayoutIndex could update its index after // `performOptimizedStructLayout` @@ -2055,11 +2059,9 @@ void coro::normalizeCoroutine(Function &F, coro::Shape &Shape, rewritePHIs(F); } -void coro::buildCoroutineFrame( - Function &F, Shape &Shape, - const std::function &MaterializableCallback) { +void coro::BaseABI::buildCoroutineFrame(bool OptimizeFrame) { SuspendCrossingInfo Checker(F, Shape.CoroSuspends, Shape.CoroEnds); - doRematerializations(F, Checker, MaterializableCallback); + doRematerializations(F, Checker, IsMaterializable); const DominatorTree DT(F); if (Shape.ABI != coro::ABI::Async && Shape.ABI != coro::ABI::Retcon && @@ -2088,7 +2090,7 @@ void coro::buildCoroutineFrame( // Build frame FrameDataInfo FrameData(Spills, Allocas); - Shape.FrameTy = buildFrameType(F, Shape, FrameData); + Shape.FrameTy = buildFrameType(F, Shape, FrameData, OptimizeFrame); Shape.FramePtr = Shape.CoroBegin; // For now, this works for C++ programs only. buildFrameDebugInfo(F, Shape, FrameData); diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h index fcbd31878bdea..a0b52063aca10 100644 --- a/llvm/lib/Transforms/Coroutines/CoroInternal.h +++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h @@ -11,10 +11,10 @@ #ifndef LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H #define LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H -#include "CoroInstr.h" -#include "CoroShape.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/Transforms/Coroutines/CoroInstr.h" +#include "llvm/Transforms/Coroutines/CoroShape.h" namespace llvm { @@ -62,9 +62,6 @@ struct LowererBase { bool defaultMaterializable(Instruction &V); void normalizeCoroutine(Function &F, coro::Shape &Shape, TargetTransformInfo &TTI); -void buildCoroutineFrame( - Function &F, Shape &Shape, - const std::function &MaterializableCallback); CallInst *createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, TargetTransformInfo &TTI, ArrayRef Arguments, IRBuilder<> &); diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp index 8ea460badaad5..1c29c57e88ad4 100644 --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -19,7 +19,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Coroutines/CoroSplit.h" -#include "CoroInstr.h" #include "CoroInternal.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PriorityWorklist.h" @@ -62,6 +61,8 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Coroutines/ABI.h" +#include "llvm/Transforms/Coroutines/CoroInstr.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/CallGraphUpdater.h" @@ -1748,9 +1749,9 @@ CallInst *coro::createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, return TailCall; } -static void splitAsyncCoroutine(Function &F, coro::Shape &Shape, - SmallVectorImpl &Clones, - TargetTransformInfo &TTI) { +void coro::AsyncABI::splitCoroutine(Function &F, coro::Shape &Shape, + SmallVectorImpl &Clones, + TargetTransformInfo &TTI) { assert(Shape.ABI == coro::ABI::Async); assert(Clones.empty()); // Reset various things that the optimizer might have decided it @@ -1843,9 +1844,9 @@ static void splitAsyncCoroutine(Function &F, coro::Shape &Shape, } } -static void splitRetconCoroutine(Function &F, coro::Shape &Shape, - SmallVectorImpl &Clones, - TargetTransformInfo &TTI) { +void coro::AnyRetconABI::splitCoroutine(Function &F, coro::Shape &Shape, + SmallVectorImpl &Clones, + TargetTransformInfo &TTI) { assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce); assert(Clones.empty()); @@ -2013,26 +2014,28 @@ static bool hasSafeElideCaller(Function &F) { return false; } -static coro::Shape -splitCoroutine(Function &F, SmallVectorImpl &Clones, - TargetTransformInfo &TTI, bool OptimizeFrame, - std::function MaterializableCallback) { - PrettyStackTraceFunction prettyStackTrace(F); +void coro::SwitchABI::splitCoroutine(Function &F, coro::Shape &Shape, + SmallVectorImpl &Clones, + TargetTransformInfo &TTI) { + SwitchCoroutineSplitter::split(F, Shape, Clones, TTI); +} - // The suspend-crossing algorithm in buildCoroutineFrame get tripped - // up by uses in unreachable blocks, so remove them as a first pass. - removeUnreachableBlocks(F); +static void doSplitCoroutine(Function &F, SmallVectorImpl &Clones, + coro::BaseABI &ABI, TargetTransformInfo &TTI, + bool OptimizeFrame) { + PrettyStackTraceFunction prettyStackTrace(F); - coro::Shape Shape(F, OptimizeFrame); - if (!Shape.CoroBegin) - return Shape; + auto &Shape = ABI.Shape; + assert(Shape.CoroBegin); lowerAwaitSuspends(F, Shape); simplifySuspendPoints(Shape); + normalizeCoroutine(F, Shape, TTI); - buildCoroutineFrame(F, Shape, MaterializableCallback); + ABI.buildCoroutineFrame(OptimizeFrame); replaceFrameSizeAndAlignment(Shape); + bool isNoSuspendCoroutine = Shape.CoroSuspends.empty(); bool shouldCreateNoAllocVariant = !isNoSuspendCoroutine && @@ -2044,18 +2047,7 @@ splitCoroutine(Function &F, SmallVectorImpl &Clones, if (isNoSuspendCoroutine) { handleNoSuspendCoroutine(Shape); } else { - switch (Shape.ABI) { - case coro::ABI::Switch: - SwitchCoroutineSplitter::split(F, Shape, Clones, TTI); - break; - case coro::ABI::Async: - splitAsyncCoroutine(F, Shape, Clones, TTI); - break; - case coro::ABI::Retcon: - case coro::ABI::RetconOnce: - splitRetconCoroutine(F, Shape, Clones, TTI); - break; - } + ABI.splitCoroutine(F, Shape, Clones, TTI); } // Replace all the swifterror operations in the original function. @@ -2076,8 +2068,6 @@ splitCoroutine(Function &F, SmallVectorImpl &Clones, if (shouldCreateNoAllocVariant) SwitchCoroutineSplitter::createNoAllocVariant(F, Shape, Clones); - - return Shape; } static LazyCallGraph::SCC &updateCallGraphAfterCoroutineSplit( @@ -2176,8 +2166,47 @@ static void addPrepareFunction(const Module &M, Fns.push_back(PrepareFn); } +static coro::BaseABI *CreateNewABI(Function &F, coro::Shape &S) { + switch (S.ABI) { + case coro::ABI::Switch: + return new coro::SwitchABI(F, S); + case coro::ABI::Async: + return new coro::AsyncABI(F, S); + case coro::ABI::Retcon: + return new coro::AnyRetconABI(F, S); + case coro::ABI::RetconOnce: + return new coro::AnyRetconABI(F, S); + } + llvm_unreachable("Bad ABI"); +} + CoroSplitPass::CoroSplitPass(bool OptimizeFrame) - : MaterializableCallback(coro::defaultMaterializable), + : GenABI([](Function &F, coro::Shape &S) { return CreateNewABI(F, S); }), + OptimizeFrame(OptimizeFrame) {} + +static coro::BaseABI * +CreateNewABIIsMat(Function &F, coro::Shape &S, + std::function IsMatCallback) { + switch (S.ABI) { + case coro::ABI::Switch: + return new coro::SwitchABI(F, S, IsMatCallback); + case coro::ABI::Async: + return new coro::AsyncABI(F, S, IsMatCallback); + case coro::ABI::Retcon: + return new coro::AnyRetconABI(F, S, IsMatCallback); + case coro::ABI::RetconOnce: + return new coro::AnyRetconABI(F, S, IsMatCallback); + } + llvm_unreachable("Bad ABI"); +} + +// For back compatibility, constructor takes a materializable callback and +// creates a generator for an ABI with a modified materializable callback. +CoroSplitPass::CoroSplitPass(std::function IsMatCallback, + bool OptimizeFrame) + : GenABI([=](Function &F, coro::Shape &S) { + return CreateNewABIIsMat(F, S, IsMatCallback); + }), OptimizeFrame(OptimizeFrame) {} PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C, @@ -2210,12 +2239,26 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C, Function &F = N->getFunction(); LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName() << "\n"); + + // The suspend-crossing algorithm in buildCoroutineFrame gets tripped up + // by unreachable blocks, so remove them as a first pass. Remove the + // unreachable blocks before collecting intrinsics into Shape. + removeUnreachableBlocks(F); + + coro::Shape Shape(F); + if (!Shape.CoroBegin) { + Shape.invalidateCoroutine(F); + continue; + } + F.setSplittedCoroutine(); + std::unique_ptr ABI(GenABI(F, Shape)); + ABI->init(); + SmallVector Clones; - coro::Shape Shape = - splitCoroutine(F, Clones, FAM.getResult(F), - OptimizeFrame, MaterializableCallback); + auto &TTI = FAM.getResult(F); + doSplitCoroutine(F, Clones, *ABI, TTI, OptimizeFrame); CurrentSCC = &updateCallGraphAfterCoroutineSplit( *N, Shape, Clones, *CurrentSCC, CG, AM, UR, FAM); diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp index 5cc13a584aef3..f83130b22ed6e 100644 --- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp +++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp @@ -10,9 +10,7 @@ // //===----------------------------------------------------------------------===// -#include "CoroInstr.h" #include "CoroInternal.h" -#include "CoroShape.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/CallGraph.h" @@ -28,6 +26,9 @@ #include "llvm/IR/Type.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Transforms/Coroutines/ABI.h" +#include "llvm/Transforms/Coroutines/CoroInstr.h" +#include "llvm/Transforms/Coroutines/CoroShape.h" #include "llvm/Transforms/Utils/Local.h" #include #include @@ -177,17 +178,6 @@ void coro::suppressCoroAllocs(LLVMContext &Context, } } -static void clear(coro::Shape &Shape) { - Shape.CoroBegin = nullptr; - Shape.CoroEnds.clear(); - Shape.CoroSizes.clear(); - Shape.CoroSuspends.clear(); - - Shape.FrameTy = nullptr; - Shape.FramePtr = nullptr; - Shape.AllocaSpillBlock = nullptr; -} - static CoroSaveInst *createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst) { Module *M = SuspendInst->getModule(); @@ -200,13 +190,12 @@ static CoroSaveInst *createCoroSave(CoroBeginInst *CoroBegin, } // Collect "interesting" coroutine intrinsics. -void coro::Shape::buildFrom(Function &F) { +void coro::Shape::analyze(Function &F) { + clear(); + bool HasFinalSuspend = false; bool HasUnwindCoroEnd = false; size_t FinalSuspendIndex = 0; - clear(*this); - SmallVector CoroFrames; - SmallVector UnusedCoroSaves; for (Instruction &I : instructions(F)) { // FIXME: coro_await_suspend_* are not proper `IntrinisicInst`s @@ -298,8 +287,58 @@ void coro::Shape::buildFrom(Function &F) { } } - // If for some reason, we were not able to find coro.begin, bailout. - if (!CoroBegin) { + // If there is no CoroBegin then this is not a coroutine. + if (!CoroBegin) + return; + + // Determination of ABI and initializing lowering info + auto Id = CoroBegin->getId(); + auto IntrID = Id->getIntrinsicID(); + if (IntrID == Intrinsic::coro_id) { + ABI = coro::ABI::Switch; + SwitchLowering.HasFinalSuspend = HasFinalSuspend; + SwitchLowering.HasUnwindCoroEnd = HasUnwindCoroEnd; + + auto SwitchId = getSwitchCoroId(); + SwitchLowering.ResumeSwitch = nullptr; + SwitchLowering.PromiseAlloca = SwitchId->getPromise(); + SwitchLowering.ResumeEntryBlock = nullptr; + + // Move final suspend to the last element in the CoroSuspends vector. + if (SwitchLowering.HasFinalSuspend && + FinalSuspendIndex != CoroSuspends.size() - 1) + std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back()); + } else if (IntrID == Intrinsic::coro_id_async) { + ABI = coro::ABI::Async; + auto *AsyncId = getAsyncCoroId(); + AsyncId->checkWellFormed(); + AsyncLowering.Context = AsyncId->getStorage(); + AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex(); + AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize(); + AsyncLowering.ContextAlignment = AsyncId->getStorageAlignment().value(); + AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer(); + AsyncLowering.AsyncCC = F.getCallingConv(); + } else if (IntrID == Intrinsic::coro_id_retcon || + IntrID == Intrinsic::coro_id_retcon_once) { + ABI = IntrID == Intrinsic::coro_id_retcon ? coro::ABI::Retcon + : coro::ABI::RetconOnce; + auto ContinuationId = getRetconCoroId(); + ContinuationId->checkWellFormed(); + auto Prototype = ContinuationId->getPrototype(); + RetconLowering.ResumePrototype = Prototype; + RetconLowering.Alloc = ContinuationId->getAllocFunction(); + RetconLowering.Dealloc = ContinuationId->getDeallocFunction(); + RetconLowering.ReturnBlock = nullptr; + RetconLowering.IsFrameInlineInStorage = false; + } else { + llvm_unreachable("coro.begin is not dependent on a coro.id call"); + } +} + +// If for some reason, we were not able to find coro.begin, bailout. +void coro::Shape::invalidateCoroutine(Function &F) { + assert(!CoroBegin); + { // Replace coro.frame which are supposed to be lowered to the result of // coro.begin with undef. auto *Undef = UndefValue::get(PointerType::get(F.getContext(), 0)); @@ -320,22 +359,13 @@ void coro::Shape::buildFrom(Function &F) { // Replace all coro.ends with unreachable instruction. for (AnyCoroEndInst *CE : CoroEnds) changeToUnreachable(CE); - - return; } +} - auto Id = CoroBegin->getId(); - switch (auto IdIntrinsic = Id->getIntrinsicID()) { - case Intrinsic::coro_id: { - auto SwitchId = cast(Id); - this->ABI = coro::ABI::Switch; - this->SwitchLowering.HasFinalSuspend = HasFinalSuspend; - this->SwitchLowering.HasUnwindCoroEnd = HasUnwindCoroEnd; - this->SwitchLowering.ResumeSwitch = nullptr; - this->SwitchLowering.PromiseAlloca = SwitchId->getPromise(); - this->SwitchLowering.ResumeEntryBlock = nullptr; - - for (auto *AnySuspend : CoroSuspends) { +void coro::SwitchABI::init() { + assert(Shape.ABI == coro::ABI::Switch); + { + for (auto *AnySuspend : Shape.CoroSuspends) { auto Suspend = dyn_cast(AnySuspend); if (!Suspend) { #ifndef NDEBUG @@ -345,43 +375,27 @@ void coro::Shape::buildFrom(Function &F) { } if (!Suspend->getCoroSave()) - createCoroSave(CoroBegin, Suspend); + createCoroSave(Shape.CoroBegin, Suspend); } - break; } - case Intrinsic::coro_id_async: { - auto *AsyncId = cast(Id); - AsyncId->checkWellFormed(); - this->ABI = coro::ABI::Async; - this->AsyncLowering.Context = AsyncId->getStorage(); - this->AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex(); - this->AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize(); - this->AsyncLowering.ContextAlignment = - AsyncId->getStorageAlignment().value(); - this->AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer(); - this->AsyncLowering.AsyncCC = F.getCallingConv(); - break; - }; - case Intrinsic::coro_id_retcon: - case Intrinsic::coro_id_retcon_once: { - auto ContinuationId = cast(Id); - ContinuationId->checkWellFormed(); - this->ABI = (IdIntrinsic == Intrinsic::coro_id_retcon - ? coro::ABI::Retcon - : coro::ABI::RetconOnce); - auto Prototype = ContinuationId->getPrototype(); - this->RetconLowering.ResumePrototype = Prototype; - this->RetconLowering.Alloc = ContinuationId->getAllocFunction(); - this->RetconLowering.Dealloc = ContinuationId->getDeallocFunction(); - this->RetconLowering.ReturnBlock = nullptr; - this->RetconLowering.IsFrameInlineInStorage = false; + Shape.tidyCoroutine(); +} + +void coro::AsyncABI::init() { + assert(Shape.ABI == coro::ABI::Async); + Shape.tidyCoroutine(); +} + +void coro::AnyRetconABI::init() { + assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce); + { // Determine the result value types, and make sure they match up with // the values passed to the suspends. - auto ResultTys = getRetconResultTypes(); - auto ResumeTys = getRetconResumeTypes(); + auto ResultTys = Shape.getRetconResultTypes(); + auto ResumeTys = Shape.getRetconResumeTypes(); - for (auto *AnySuspend : CoroSuspends) { + for (auto *AnySuspend : Shape.CoroSuspends) { auto Suspend = dyn_cast(AnySuspend); if (!Suspend) { #ifndef NDEBUG @@ -408,7 +422,7 @@ void coro::Shape::buildFrom(Function &F) { #ifndef NDEBUG Suspend->dump(); - Prototype->getFunctionType()->dump(); + Shape.RetconLowering.ResumePrototype->getFunctionType()->dump(); #endif report_fatal_error("argument to coro.suspend.retcon does not " "match corresponding prototype function result"); @@ -417,14 +431,14 @@ void coro::Shape::buildFrom(Function &F) { if (SI != SE || RI != RE) { #ifndef NDEBUG Suspend->dump(); - Prototype->getFunctionType()->dump(); + Shape.RetconLowering.ResumePrototype->getFunctionType()->dump(); #endif report_fatal_error("wrong number of arguments to coro.suspend.retcon"); } // Check that the result type of the suspend matches the resume types. Type *SResultTy = Suspend->getType(); - ArrayRef SuspendResultTys; + ArrayRef SuspendResultTys; if (SResultTy->isVoidTy()) { // leave as empty array } else if (auto SResultStructTy = dyn_cast(SResultTy)) { @@ -436,7 +450,7 @@ void coro::Shape::buildFrom(Function &F) { if (SuspendResultTys.size() != ResumeTys.size()) { #ifndef NDEBUG Suspend->dump(); - Prototype->getFunctionType()->dump(); + Shape.RetconLowering.ResumePrototype->getFunctionType()->dump(); #endif report_fatal_error("wrong number of results from coro.suspend.retcon"); } @@ -444,32 +458,25 @@ void coro::Shape::buildFrom(Function &F) { if (SuspendResultTys[I] != ResumeTys[I]) { #ifndef NDEBUG Suspend->dump(); - Prototype->getFunctionType()->dump(); + Shape.RetconLowering.ResumePrototype->getFunctionType()->dump(); #endif report_fatal_error("result from coro.suspend.retcon does not " "match corresponding prototype function param"); } } } - break; } - default: - llvm_unreachable("coro.begin is not dependent on a coro.id call"); - } + Shape.tidyCoroutine(); +} +void coro::Shape::tidyCoroutine() { // The coro.free intrinsic is always lowered to the result of coro.begin. for (CoroFrameInst *CF : CoroFrames) { CF->replaceAllUsesWith(CoroBegin); CF->eraseFromParent(); } - // Move final suspend to be the last element in the CoroSuspends vector. - if (ABI == coro::ABI::Switch && - SwitchLowering.HasFinalSuspend && - FinalSuspendIndex != CoroSuspends.size() - 1) - std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back()); - // Remove orphaned coro.saves. for (CoroSaveInst *CoroSave : UnusedCoroSaves) CoroSave->eraseFromParent(); diff --git a/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp b/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp index 708e8734175f9..c3ea0977d4211 100644 --- a/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp +++ b/llvm/lib/Transforms/Coroutines/MaterializationUtils.cpp @@ -9,12 +9,13 @@ // This file contains classes used to materialize insts after suspends points. //===----------------------------------------------------------------------===// -#include "MaterializationUtils.h" -#include "SpillUtils.h" +#include "llvm/Transforms/Coroutines/MaterializationUtils.h" +#include "CoroInternal.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instruction.h" +#include "llvm/Transforms/Coroutines/SpillUtils.h" #include using namespace llvm; diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp index f213ac1c8d7d5..e85c43b3fbc95 100644 --- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp +++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "SpillUtils.h" +#include "llvm/Transforms/Coroutines/SpillUtils.h" #include "llvm/Analysis/CFG.h" #include "llvm/Analysis/PtrUseVisitor.h" #include "llvm/IR/CFG.h" diff --git a/llvm/lib/Transforms/Coroutines/SuspendCrossingInfo.cpp b/llvm/lib/Transforms/Coroutines/SuspendCrossingInfo.cpp index 84699e653db60..f18f23306befb 100644 --- a/llvm/lib/Transforms/Coroutines/SuspendCrossingInfo.cpp +++ b/llvm/lib/Transforms/Coroutines/SuspendCrossingInfo.cpp @@ -12,7 +12,7 @@ // ptrs in the BlockToIndexMapping. //===----------------------------------------------------------------------===// -#include "SuspendCrossingInfo.h" +#include "llvm/Transforms/Coroutines/SuspendCrossingInfo.h" // The "coro-suspend-crossing" flag is very noisy. There is another debug type, // "coro-frame", which results in leaner debug spew. diff --git a/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp b/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp index da78c151e7f68..c4d80c47023b8 100644 --- a/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp +++ b/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp @@ -11,6 +11,7 @@ #include "llvm/Passes/PassBuilder.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Testing/Support/Error.h" +#include "llvm/Transforms/Coroutines/ABI.h" #include "llvm/Transforms/Coroutines/CoroSplit.h" #include "gtest/gtest.h"