From da8ecdc37707346b2fd482610dbc3b6fdc67032f Mon Sep 17 00:00:00 2001 From: John McCall Date: Thu, 29 Jun 2023 14:49:59 -0400 Subject: [PATCH 1/4] [NFC] Add a method to just ask if a tuple AP vanishes under substitution --- include/swift/SIL/AbstractionPattern.h | 4 ++++ include/swift/SIL/AbstractionPatternGenerators.h | 5 +++++ lib/SIL/IR/AbstractionPattern.cpp | 10 +++++++--- lib/SILGen/ResultPlan.cpp | 2 +- lib/SILGen/SILGenApply.cpp | 3 +-- lib/SILGen/SILGenStmt.cpp | 3 +-- 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/include/swift/SIL/AbstractionPattern.h b/include/swift/SIL/AbstractionPattern.h index ddc33c5036588..cbc38b075a57a 100644 --- a/include/swift/SIL/AbstractionPattern.h +++ b/include/swift/SIL/AbstractionPattern.h @@ -1344,6 +1344,10 @@ class AbstractionPattern { /// returned element is the pattern type of the expansion. Optional getVanishingTupleElementPatternType() const; + /// Does this tuple type vanish, i.e. is it flattened to a singleton + /// non-expansion element under substitution? + bool doesTupleVanish() const; + static AbstractionPattern projectTupleElementType(const AbstractionPattern *base, size_t index) { return base->getTupleElementType(index); diff --git a/include/swift/SIL/AbstractionPatternGenerators.h b/include/swift/SIL/AbstractionPatternGenerators.h index bc5684988a9b2..0c472812fe2f0 100644 --- a/include/swift/SIL/AbstractionPatternGenerators.h +++ b/include/swift/SIL/AbstractionPatternGenerators.h @@ -224,6 +224,11 @@ class TupleElementGenerator { return origEltIndex == numOrigElts; } + /// Does the entire original tuple vanish? + bool doesOrigTupleVanish() const { + return origTupleVanishes; + } + /// Advance to the next orig element. void advance() { assert(!isFinished()); diff --git a/lib/SIL/IR/AbstractionPattern.cpp b/lib/SIL/IR/AbstractionPattern.cpp index 1188526024708..d8957b5ad3e25 100644 --- a/lib/SIL/IR/AbstractionPattern.cpp +++ b/lib/SIL/IR/AbstractionPattern.cpp @@ -314,7 +314,7 @@ bool AbstractionPattern::matchesTuple(CanType substType) const { return false; LLVM_FALLTHROUGH; case Kind::Tuple: { - if (getVanishingTupleElementPatternType()) { + if (doesTupleVanish()) { // TODO: recurse into elements. return true; } @@ -480,6 +480,11 @@ bool AbstractionPattern::doesTupleContainPackExpansionType() const { llvm_unreachable("bad kind"); } +bool AbstractionPattern::doesTupleVanish() const { + assert(isTuple()); + return getVanishingTupleElementPatternType().hasValue(); +} + Optional AbstractionPattern::getVanishingTupleElementPatternType() const { if (!isTuple()) return None; @@ -552,8 +557,7 @@ TupleElementGenerator::TupleElementGenerator( assert(origTupleType.isTuple()); assert(origTupleType.matchesTuple(substType)); - origTupleVanishes = - origTupleType.getVanishingTupleElementPatternType().hasValue(); + origTupleVanishes = origTupleType.doesTupleVanish(); origTupleTypeIsOpaque = origTupleType.isOpaqueTuple(); numOrigElts = origTupleType.getNumTupleElements(); diff --git a/lib/SILGen/ResultPlan.cpp b/lib/SILGen/ResultPlan.cpp index 4faa37d0640d2..2f35ea29dddd8 100644 --- a/lib/SILGen/ResultPlan.cpp +++ b/lib/SILGen/ResultPlan.cpp @@ -1326,7 +1326,7 @@ ResultPlanPtr ResultPlanBuilder::buildForTuple(Initialization *init, // emit directly into the initialization. If the orig tuple vanishes, // that counts as the initialization being splittable. if (init) { - bool vanishes = origType.getVanishingTupleElementPatternType().hasValue(); + bool vanishes = origType.doesTupleVanish(); if (vanishes || init->canSplitIntoTupleElements()) { return ResultPlanPtr( new TupleInitializationResultPlan(*this, init, origType, substType, diff --git a/lib/SILGen/SILGenApply.cpp b/lib/SILGen/SILGenApply.cpp index 8212dbd7b7cc3..97b04b7611b48 100644 --- a/lib/SILGen/SILGenApply.cpp +++ b/lib/SILGen/SILGenApply.cpp @@ -3416,8 +3416,7 @@ class ArgEmitter { // If the original parameter type is a vanishing tuple, we want to emit // this as if the argument source was wrapped in an extra level of // tuple literal. - bool origTupleVanishes = - origParamType.getVanishingTupleElementPatternType().hasValue(); + bool origTupleVanishes = origParamType.doesTupleVanish(); auto substType = arg.getSubstRValueType(); diff --git a/lib/SILGen/SILGenStmt.cpp b/lib/SILGen/SILGenStmt.cpp index c6da83b620820..6856751c7091d 100644 --- a/lib/SILGen/SILGenStmt.cpp +++ b/lib/SILGen/SILGenStmt.cpp @@ -588,8 +588,7 @@ prepareIndirectResultInit(SILGenFunction &SGF, SILLocation loc, TupleInitialization *tupleInit = nullptr; SmallVector singletonEltInit; - bool vanishes = - origResultType.getVanishingTupleElementPatternType().hasValue(); + bool vanishes = origResultType.doesTupleVanish(); if (!vanishes) { auto resultTupleType = cast(resultType); tupleInit = new TupleInitialization(resultTupleType); From bf9632e61b7ec2ee3680e64ebdb76d408601c00c Mon Sep 17 00:00:00 2001 From: John McCall Date: Thu, 29 Jun 2023 19:33:16 -0400 Subject: [PATCH 2/4] Prevent SimpleGeneratorRef from being used as the copy constructor. --- include/swift/Basic/Generators.h | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/include/swift/Basic/Generators.h b/include/swift/Basic/Generators.h index 748711b0eec7f..3660b3bca895e 100644 --- a/include/swift/Basic/Generators.h +++ b/include/swift/Basic/Generators.h @@ -146,6 +146,10 @@ class ArrayRefGenerator { } }; +namespace generator_details { +template struct is_simple_generator_ref; +} + /// An abstracting reference to an existing generator. /// /// The implementation of this type holds the reference to the existing @@ -182,7 +186,8 @@ class SimpleGeneratorRef { constexpr SimpleGeneratorRef() : vtable(nullptr), pointer(nullptr) {} template - constexpr SimpleGeneratorRef(G &generator) + constexpr SimpleGeneratorRef(G &generator, + typename std::enable_if::value, bool>::type = false) : vtable(&VTableImpl::vtable), pointer(&generator) {} /// Test whether this generator ref was initialized with a @@ -212,6 +217,19 @@ class SimpleGeneratorRef { } }; +namespace generator_details { + +template +struct is_simple_generator_ref> { + static constexpr bool value = true; +}; +template +struct is_simple_generator_ref { + static constexpr bool value = false; +}; + +} + } // end namespace swift #endif From d47bfe58144568ff6564412461898b468450f545 Mon Sep 17 00:00:00 2001 From: John McCall Date: Thu, 29 Jun 2023 19:34:00 -0400 Subject: [PATCH 3/4] Fix some SILArgument infrastructure for pack results. Getting this right convinces the memory verifier to not complain about untouched empty pack result arguments, which should be innocuous. --- .../Sources/SIL/Argument.swift | 27 ++++++++++++-- .../Sources/SIL/Effects.swift | 4 +-- .../Sources/SIL/Function.swift | 3 -- include/swift/SIL/SILArgument.h | 13 ------- include/swift/SIL/SILArgumentConvention.h | 22 ++++++++++++ include/swift/SIL/SILBridging.h | 2 +- include/swift/SIL/SILFunctionConventions.h | 1 - lib/SIL/IR/SILArgument.cpp | 36 +++++++++++++++++++ 8 files changed, 85 insertions(+), 23 deletions(-) diff --git a/SwiftCompilerSources/Sources/SIL/Argument.swift b/SwiftCompilerSources/Sources/SIL/Argument.swift index 25003c2c84e56..2ee54d1b386f3 100644 --- a/SwiftCompilerSources/Sources/SIL/Argument.swift +++ b/SwiftCompilerSources/Sources/SIL/Argument.swift @@ -137,11 +137,17 @@ public enum ArgumentConvention { /// indirectly is recorded in the pack type. case packGuaranteed + /// This argument is a pack of indirect return value addresses. The + /// addresses are stored in the pack by the caller and read out by the + /// callee; within the callee, they are individually treated like + /// indirectOut arguments. + case packOut + public var isIndirect: Bool { switch self { case .indirectIn, .indirectInGuaranteed, .indirectInout, .indirectInoutAliasable, .indirectOut, - .packInout, .packOwned, .packGuaranteed: + .packOut, .packInout, .packOwned, .packGuaranteed: return true case .directOwned, .directUnowned, .directGuaranteed: return false @@ -155,7 +161,19 @@ public enum ArgumentConvention { return true case .directOwned, .directUnowned, .directGuaranteed, .indirectInout, .indirectInoutAliasable, .indirectOut, - .packInout: + .packOut, .packInout: + return false + } + } + + public var isIndirectOut: Bool { + switch self { + case .indirectOut, .packOut: + return true + case .indirectInGuaranteed, .directGuaranteed, .packGuaranteed, + .indirectIn, .directOwned, .directUnowned, + .indirectInout, .indirectInoutAliasable, + .packInout, .packOwned: return false } } @@ -166,7 +184,7 @@ public enum ArgumentConvention { return true case .indirectIn, .directOwned, .directUnowned, .indirectInout, .indirectInoutAliasable, .indirectOut, - .packInout, .packOwned: + .packOut, .packInout, .packOwned: return false } } @@ -177,6 +195,7 @@ public enum ArgumentConvention { .indirectOut, .indirectInGuaranteed, .indirectInout, + .packOut, .packInout, .packOwned, .packGuaranteed: @@ -203,6 +222,7 @@ public enum ArgumentConvention { .directUnowned, .directGuaranteed, .directOwned, + .packOut, .packOwned, .packGuaranteed: return false @@ -229,6 +249,7 @@ extension BridgedArgumentConvention { case .Direct_Owned: return .directOwned case .Direct_Unowned: return .directUnowned case .Direct_Guaranteed: return .directGuaranteed + case .Pack_Out: return .packOut case .Pack_Inout: return .packInout case .Pack_Owned: return .packOwned case .Pack_Guaranteed: return .packGuaranteed diff --git a/SwiftCompilerSources/Sources/SIL/Effects.swift b/SwiftCompilerSources/Sources/SIL/Effects.swift index 4bc74c50328ac..52e1aa2c3b7b4 100644 --- a/SwiftCompilerSources/Sources/SIL/Effects.swift +++ b/SwiftCompilerSources/Sources/SIL/Effects.swift @@ -143,7 +143,7 @@ extension Function { if convention.isIndirectIn { // Even a `[readnone]` function can read from an indirect argument. result.memory.read = true - } else if convention == .indirectOut { + } else if convention.isIndirectOut { // Even `[readnone]` and `[readonly]` functions write to indirect results. result.memory.write = true } @@ -546,7 +546,7 @@ public struct SideEffects : CustomStringConvertible, NoReflectionChildren { case .indirectInGuaranteed, .packGuaranteed: result.memory.write = false result.ownership.destroy = false - case .indirectOut, .packInout: + case .indirectOut, .packOut, .packInout: result.memory.read = false result.ownership.copy = false result.ownership.destroy = false diff --git a/SwiftCompilerSources/Sources/SIL/Function.swift b/SwiftCompilerSources/Sources/SIL/Function.swift index 9e86d62ba7c57..13c5d857af480 100644 --- a/SwiftCompilerSources/Sources/SIL/Function.swift +++ b/SwiftCompilerSources/Sources/SIL/Function.swift @@ -83,9 +83,6 @@ final public class Function : CustomStringConvertible, HasShortDescription, Hash public var resultType: Type { bridged.getSILResultType().type } public func getArgumentConvention(for argumentIndex: Int) -> ArgumentConvention { - if argumentIndex < numIndirectResultArguments { - return .indirectOut - } return bridged.getSILArgumentConvention(argumentIndex).convention } diff --git a/include/swift/SIL/SILArgument.h b/include/swift/SIL/SILArgument.h index 078d9d7784732..bf7154e3d5a58 100644 --- a/include/swift/SIL/SILArgument.h +++ b/include/swift/SIL/SILArgument.h @@ -28,19 +28,6 @@ class SILPhiArgument; class SILUndef; class TermInst; -// Map an argument index onto a SILArgumentConvention. -inline SILArgumentConvention -SILFunctionConventions::getSILArgumentConvention(unsigned index) const { - assert(index <= getNumSILArguments()); - if (index < getNumIndirectSILResults()) { - assert(silConv.loweredAddresses); - return SILArgumentConvention::Indirect_Out; - } else { - auto param = funcTy->getParameters()[index - getNumIndirectSILResults()]; - return SILArgumentConvention(param.getConvention()); - } -} - struct SILArgumentKind { enum innerty : std::underlying_type::type { #define ARGUMENT(ID, PARENT) ID = unsigned(SILNodeKind::ID), diff --git a/include/swift/SIL/SILArgumentConvention.h b/include/swift/SIL/SILArgumentConvention.h index ed626fc6f9b96..cb1ecb9aee171 100644 --- a/include/swift/SIL/SILArgumentConvention.h +++ b/include/swift/SIL/SILArgumentConvention.h @@ -164,6 +164,28 @@ struct SILArgumentConvention { } llvm_unreachable("covered switch isn't covered?!"); } + + /// Returns true if \p Value is an indirect-out parameter. + bool isIndirectOutParameter() { + switch (Value) { + case SILArgumentConvention::Indirect_Out: + case SILArgumentConvention::Pack_Out: + return true; + + case SILArgumentConvention::Indirect_In: + case SILArgumentConvention::Indirect_In_Guaranteed: + case SILArgumentConvention::Indirect_Inout: + case SILArgumentConvention::Indirect_InoutAliasable: + case SILArgumentConvention::Direct_Unowned: + case SILArgumentConvention::Direct_Guaranteed: + case SILArgumentConvention::Direct_Owned: + case SILArgumentConvention::Pack_Inout: + case SILArgumentConvention::Pack_Owned: + case SILArgumentConvention::Pack_Guaranteed: + return false; + } + llvm_unreachable("covered switch isn't covered?!"); + } }; } // namespace swift diff --git a/include/swift/SIL/SILBridging.h b/include/swift/SIL/SILBridging.h index b1c52ff2cab35..a4b3e90ec727c 100644 --- a/include/swift/SIL/SILBridging.h +++ b/include/swift/SIL/SILBridging.h @@ -212,7 +212,7 @@ struct BridgedFunction { BridgedArgumentConvention getSILArgumentConvention(SwiftInt idx) const { swift::SILFunctionConventions conv(getFunction()->getConventionsInContext()); - return castToArgumentConvention(swift::SILArgumentConvention(conv.getParamInfoForSILArg(idx).getConvention())); + return castToArgumentConvention(conv.getSILArgumentConvention(idx)); } swift::SILType getSILResultType() const { diff --git a/include/swift/SIL/SILFunctionConventions.h b/include/swift/SIL/SILFunctionConventions.h index 44e8d0f3b827c..686e6d315c6fd 100644 --- a/include/swift/SIL/SILFunctionConventions.h +++ b/include/swift/SIL/SILFunctionConventions.h @@ -409,7 +409,6 @@ class SILFunctionConventions { /// Return the SIL argument convention of apply/entry argument at /// the given argument index. SILArgumentConvention getSILArgumentConvention(unsigned index) const; - // See SILArgument.h. /// Return the SIL type of the apply/entry argument at the given index. SILType getSILArgumentType(unsigned index, diff --git a/lib/SIL/IR/SILArgument.cpp b/lib/SIL/IR/SILArgument.cpp index 0f8821671a62c..ce85acf41ee95 100644 --- a/lib/SIL/IR/SILArgument.cpp +++ b/lib/SIL/IR/SILArgument.cpp @@ -72,6 +72,42 @@ SILParameterInfo SILFunctionArgument::getKnownParameterInfo() const { return getFunction()->getConventions().getParamInfoForSILArg(getIndex()); } +SILArgumentConvention +SILFunctionConventions::getSILArgumentConvention(unsigned index) const { + assert(index < getNumSILArguments()); + + // If the argument is a parameter, index into the parameters. + if (index >= getNumIndirectSILResults()) { + auto param = funcTy->getParameters()[index - getNumIndirectSILResults()]; + return SILArgumentConvention(param.getConvention()); + } + + // If it's an indirect result, it could be either Pack_Out or + // Indirect_Out. + + // Handle the common case of a function with no pack results. + if (funcTy->getNumPackResults() == 0) { + assert(silConv.loweredAddresses); + return SILArgumentConvention::Indirect_Out; + } + + // Otherwise, we need to index into the indirect results to figure out + // whether the result is a pack or not, and unfortunately that is not a + // linear algorithm. + for (auto result : getIndirectSILResults()) { + if (index == 0) { + if (result.getConvention() == ResultConvention::Indirect) { + assert(silConv.loweredAddresses); + return SILArgumentConvention::Indirect_Out; + } else { + assert(result.getConvention() == ResultConvention::Pack); + return SILArgumentConvention::Pack_Out; + } + } + index--; + } + llvm_unreachable("mismatch with getNumIndirectSILResults()?"); +} //===----------------------------------------------------------------------===// // SILBlockArgument From 90b631f7980825bb10cf275754ce527f274980b5 Mon Sep 17 00:00:00 2001 From: John McCall Date: Thu, 29 Jun 2023 22:20:29 -0400 Subject: [PATCH 4/4] Handle vanishing and variadic tuple results in reabstraction thunks. Fixes rdar://110391963 --- lib/SILGen/Cleanup.cpp | 32 + lib/SILGen/Cleanup.h | 3 + lib/SILGen/SILGenFunction.cpp | 2 +- lib/SILGen/SILGenFunction.h | 20 + lib/SILGen/SILGenPack.cpp | 79 + lib/SILGen/SILGenPoly.cpp | 1636 +++++++++++++---- lib/SILGen/TupleGenerators.h | 389 ++++ ...adic-generic-reabstract-tuple-result.swift | 327 ++++ 8 files changed, 2148 insertions(+), 340 deletions(-) create mode 100644 lib/SILGen/TupleGenerators.h create mode 100644 test/SILGen/variadic-generic-reabstract-tuple-result.swift diff --git a/lib/SILGen/Cleanup.cpp b/lib/SILGen/Cleanup.cpp index 9f12dec25ec24..005d3a7cf68e1 100644 --- a/lib/SILGen/Cleanup.cpp +++ b/lib/SILGen/Cleanup.cpp @@ -499,3 +499,35 @@ CleanupCloner::cloneForRemainingPackComponents(SILValue packAddr, firstComponentIndex); return ManagedValue(packAddr, cleanup); } + +ManagedValue +CleanupCloner::cloneForRemainingTupleComponents(SILValue tupleAddr, + CanPackType inducedPackType, + unsigned firstComponentIndex) const { + if (isLValue) { + return ManagedValue::forLValue(tupleAddr); + } + + if (!hasCleanup) { + return ManagedValue::forUnmanaged(tupleAddr); + } + + assert(!writebackBuffer.has_value()); + bool isTrivial = true; + auto tupleTy = tupleAddr->getType().castTo(); + for (auto eltTy : tupleTy.getElementTypes().slice(firstComponentIndex)) { + if (!SILType::getPrimitiveObjectType(eltTy).isTrivial(SGF.F)) { + isTrivial = false; + break; + } + } + + if (isTrivial) + return ManagedValue::forUnmanaged(tupleAddr); + + auto cleanup = + SGF.enterDestroyRemainingTupleElementsCleanup(tupleAddr, + inducedPackType, + firstComponentIndex); + return ManagedValue(tupleAddr, cleanup); +} diff --git a/lib/SILGen/Cleanup.h b/lib/SILGen/Cleanup.h index f99853031da78..4c8c7e4598f67 100644 --- a/lib/SILGen/Cleanup.h +++ b/lib/SILGen/Cleanup.h @@ -349,6 +349,9 @@ class CleanupCloner { ManagedValue cloneForRemainingPackComponents(SILValue packAddr, CanPackType formalPackType, unsigned firstComponentIndex) const; + ManagedValue cloneForRemainingTupleComponents(SILValue tupleAddr, + CanPackType inducedPackType, + unsigned firstComponentIndex) const; static void getClonersForRValue(SILGenFunction &SGF, const RValue &rvalue, diff --git a/lib/SILGen/SILGenFunction.cpp b/lib/SILGen/SILGenFunction.cpp index 5da8e1cfcc821..b0b88e3c84e52 100644 --- a/lib/SILGen/SILGenFunction.cpp +++ b/lib/SILGen/SILGenFunction.cpp @@ -990,7 +990,7 @@ SILGenFunction::emitClosureValue(SILLocation loc, SILDeclRef constant, // Get the lowered AST types: // - the original type - auto origFormalType = AbstractionPattern(constantInfo.LoweredType); + auto origFormalType = AbstractionPattern(subs, constantInfo.LoweredType); // - the substituted type auto substFormalType = expectedType; diff --git a/lib/SILGen/SILGenFunction.h b/lib/SILGen/SILGenFunction.h index 49f7ba54215b2..b07a1cc1cad56 100644 --- a/lib/SILGen/SILGenFunction.h +++ b/lib/SILGen/SILGenFunction.h @@ -2496,6 +2496,13 @@ class LLVM_LIBRARY_VISIBILITY SILGenFunction unsigned componentIndex, SILValue currentIndexWithinComponent); + /// Enter a cleanup to destroy all of the components in a tuple starting + /// at a particular component index. + CleanupHandle + enterDestroyRemainingTupleElementsCleanup(SILValue addr, + CanPackType inducedPackType, + unsigned componentIndex); + /// Copy the elements of a pack, which must consist of a single pack expansion, /// into a tuple value having the same pack expansion and its sole element type. void copyPackElementsToTuple(SILLocation loc, SILValue tupleAddr, SILValue pack, @@ -2716,6 +2723,19 @@ class LLVM_LIBRARY_VISIBILITY SILGenFunction CanPackType formalPackType, unsigned firstComponentIndex = 0); + /// Emit instructions to destroy a suffix of a tuple value. + /// + /// \param tupleAddr - the address of the overall tuple value + /// \param inducedPackType - a pack type with the same shape as the + /// element types of the overall tuple value; can be null if the + /// tuple type doesn't contain pack expansions + /// \param componentIndex - the index of the first component to + /// destroy in the tuple + void emitDestroyRemainingTupleElements(SILLocation loc, + SILValue tupleAddr, + CanPackType inducedPackType, + unsigned componentIndex); + /// Emit a loop which destroys a prefix of a pack expansion component /// of a tuple value. /// diff --git a/lib/SILGen/SILGenPack.cpp b/lib/SILGen/SILGenPack.cpp index 7a6e4a19a02b6..6d48a00c04a6a 100644 --- a/lib/SILGen/SILGenPack.cpp +++ b/lib/SILGen/SILGenPack.cpp @@ -207,6 +207,36 @@ class PartialDestroyRemainingTupleCleanup : public Cleanup { } }; +/// Cleanup to destroy the remaining elements in a tuple following a +/// particular value. +class DestroyRemainingTupleElementsCleanup : public Cleanup { + SILValue Addr; + unsigned ComponentIndex; + CanPackType InducedPackType; +public: + DestroyRemainingTupleElementsCleanup(SILValue tupleAddr, + CanPackType inducedPackType, + unsigned componentIndex) + : Addr(tupleAddr), ComponentIndex(componentIndex), + InducedPackType(inducedPackType) {} + + void emit(SILGenFunction &SGF, CleanupLocation l, + ForUnwind_t forUnwind) override { + SGF.emitDestroyRemainingTupleElements(l, Addr, InducedPackType, + ComponentIndex); + } + + void dump(SILGenFunction &) const override { +#ifndef NDEBUG + llvm::errs() << "DestroyRemainingTupleElementsCleanup\n" + << "State:" << getState() << "\n" + << "Addr:" << Addr << "\n" + << "InducedPackType:" << InducedPackType << "\n" + << "ComponentIndex:" << ComponentIndex << "\n"; +#endif + } +}; + /// An ASTWalker to emit tuple values in `MaterializePackExpr` nodes. /// /// Materialized packs are emitted inside a pack expansion context before @@ -322,6 +352,17 @@ SILGenFunction::enterPartialDestroyRemainingTupleCleanup(SILValue addr, return Cleanups.getTopCleanup(); } +CleanupHandle +SILGenFunction::enterDestroyRemainingTupleElementsCleanup(SILValue addr, + CanPackType formalPackType, + unsigned componentIndex) { + Cleanups.pushCleanup(addr, + formalPackType, + componentIndex); + return Cleanups.getTopCleanup(); +} + + void SILGenFunction::emitDestroyPack(SILLocation loc, SILValue packAddr, CanPackType formalPackType, unsigned firstComponentIndex) { @@ -521,6 +562,44 @@ void SILGenFunction::emitPartialDestroyRemainingTuple(SILLocation loc, }); } +void SILGenFunction::emitDestroyRemainingTupleElements( + SILLocation loc, SILValue tupleAddr, + CanPackType inducedPackType, unsigned firstComponentIndex) { + auto tupleTy = tupleAddr->getType().castTo(); + bool containsExpansions = tupleTy->containsPackExpansionType(); + assert(!containsExpansions || inducedPackType); + + // Destroy each of the elements of the pack. + for (auto componentIndex : + range(firstComponentIndex, tupleTy->getNumElements())) { + auto eltTy = tupleAddr->getType().getTupleElementType(componentIndex); + + // We can skip this if the whole thing is trivial. + auto &eltTL = getTypeLowering(eltTy); + if (eltTL.isTrivial()) continue; + + // If it's an expansion component, emit a "partial"-destroy loop. + if (auto expansion = eltTy.getAs()) { + emitPartialDestroyRemainingTuple(loc, tupleAddr, inducedPackType, + componentIndex, /*limit*/ nullptr); + + // If it's a scalar component, project and destroy it. + } else { + SILValue eltAddr; + if (containsExpansions) { + auto packIndex = + B.createScalarPackIndex(loc, componentIndex, inducedPackType); + eltAddr = + B.createTuplePackElementAddr(loc, packIndex, tupleAddr, eltTy); + } else { + eltAddr = + B.createTupleElementAddr(loc, tupleAddr, componentIndex, eltTy); + } + B.createDestroyAddr(loc, eltAddr); + } + } +} + void SILGenFunction::copyPackElementsToTuple(SILLocation loc, SILValue tupleAddr, SILValue pack, diff --git a/lib/SILGen/SILGenPoly.cpp b/lib/SILGen/SILGenPoly.cpp index 78d3030fff342..031b23ecc29b7 100644 --- a/lib/SILGen/SILGenPoly.cpp +++ b/lib/SILGen/SILGenPoly.cpp @@ -92,6 +92,7 @@ #include "SILGenFunction.h" #include "SILGenFunctionBuilder.h" #include "Scope.h" +#include "TupleGenerators.h" #include "swift/Basic/Generators.h" #include "swift/AST/ASTMangler.h" #include "swift/AST/Decl.h" @@ -1863,6 +1864,126 @@ class TranslateArguments { } // end anonymous namespace +ManagedValue +TupleElementAddressGenerator::projectElementAddress(SILGenFunction &SGF, + SILLocation loc) { + unsigned eltIndex = getSubstElementIndex(); + + auto tupleValue = this->tupleAddr; + + CleanupCloner cloner(SGF, tupleValue); + auto tupleAddr = tupleValue.forward(SGF); + + auto eltTy = tupleAddr->getType().getTupleElementType(eltIndex); + ManagedValue eltValue; + if (!tupleContainsPackExpansion()) { + eltValue = cloner.clone( + SGF.B.createTupleElementAddr(loc, tupleAddr, eltIndex, eltTy)); + } else if (isSubstPackExpansion()) { + eltValue = cloner.cloneForTuplePackExpansionComponent(tupleAddr, + inducedPackType, + eltIndex); + } else { + auto packIndex = + SGF.B.createScalarPackIndex(loc, eltIndex, inducedPackType); + auto eltAddr = + SGF.B.createTuplePackElementAddr(loc, packIndex, tupleAddr, eltTy); + eltValue = cloner.clone(eltAddr); + } + + tupleValue = cloner.cloneForRemainingTupleComponents(tupleAddr, + inducedPackType, + eltIndex + 1); + this->tupleAddr = tupleValue; + + return eltValue; +} + +ManagedValue +ExpandedTupleInputGenerator::projectPackComponent(SILGenFunction &SGF, + SILLocation loc) { + assert(isOrigPackExpansion()); + + auto formalPackType = getFormalPackType(); + auto componentIndex = getPackComponentIndex(); + + auto packValue = getPackValue(); + auto packTy = packValue.getType().castTo(); + auto componentTy = packTy->getSILElementType(componentIndex); + + auto isComponentExpansion = componentTy.is(); + assert(isComponentExpansion == isa( + formalPackType.getElementType(componentIndex))); + + // Deactive the cleanup for the input pack value. + CleanupCloner cloner(SGF, packValue); + auto packAddr = packValue.forward(SGF); + + ManagedValue componentValue; + if (isComponentExpansion) { + // "Project" the expansion component from the pack. + // This would be a slice, but we can't currently do pack slices + // in SIL, so for now we're just returning the original pack. + componentValue = + cloner.cloneForPackPackExpansionComponent(packAddr, formalPackType, + componentIndex); + + } else { + // Project the scalar element from the pack. + auto packIndex = + SGF.B.createScalarPackIndex(loc, componentIndex, formalPackType); + auto eltAddr = + SGF.B.createPackElementGet(loc, packIndex, packAddr, componentTy); + + componentValue = cloner.clone(eltAddr); + } + + // Re-enter a cleanup for whatever pack components remain. + packValue = cloner.cloneForRemainingPackComponents(packAddr, + formalPackType, + componentIndex + 1); + updatePackValue(packValue); + + return componentValue; +} + +SILValue +ExpandedTupleInputGenerator::createPackComponentTemporary(SILGenFunction &SGF, + SILLocation loc) { + assert(isOrigPackExpansion()); + + auto formalPackType = getFormalPackType(); + auto componentIndex = getPackComponentIndex(); + + auto packValue = getPackValue(); + auto packTy = packValue.getType().castTo(); + auto componentTy = packTy->getSILElementType(componentIndex); + + auto isComponentExpansion = componentTy.is(); + assert(isComponentExpansion == isa( + formalPackType.getElementType(componentIndex))); + + auto packAddr = packValue.getLValueAddress(); + + // If we don't have a pack-expansion component, we're just making a + // single element. We don't handle the expansion case for now, because + // the reabstraction code generally needs to handle it differently + // anyway. We could if it's important, but the caller would still need + // to handle it differently. + assert(!isComponentExpansion); + + // Create the temporary. + auto temporary = + SGF.emitTemporaryAllocation(loc, componentTy.getObjectType()); + + // Write the temporary into the pack at the appropriate position. + auto packIndex = + SGF.B.createScalarPackIndex(loc, componentIndex, formalPackType); + SGF.B.createPackElementSet(loc, temporary, packIndex, packAddr); + + return temporary; +} + ManagedValue FunctionInputGenerator::projectPackComponent(SILGenFunction &SGF, SILLocation loc) { @@ -1975,8 +2096,7 @@ ManagedValue TranslateArguments::translateToPackParam( outputSubstParams[outputComponentIndex].getParameterType(); auto inputSubstType = inputParam.getSubstParam().getParameterType(); - auto inputOrigPatternType = - inputParam.getOrigType().getPackExpansionPatternType(); + auto inputOrigType = inputParam.getOrigType(); auto insertScalarIntoPack = [&](ManagedValue output) { assert(!outputComponentTy.is()); @@ -1997,7 +2117,7 @@ ManagedValue TranslateArguments::translateToPackParam( SILParameterInfo outputComponentParam(outputComponentTy.getASTType(), getScalarConventionForPackConvention(outputPackParam.getConvention())); - ManagedValue output = translateIntoSingle(inputOrigPatternType, + ManagedValue output = translateIntoSingle(inputOrigType, inputSubstType, outputOrigPatternType, outputSubstType, @@ -2006,6 +2126,7 @@ ManagedValue TranslateArguments::translateToPackParam( // Otherwise, we're starting with the input value from the pack. } else { + auto inputOrigPatternType = inputOrigType.getPackExpansionPatternType(); auto inputComponentIndex = inputParam.getPackComponentIndex(); auto inputPackValue = inputParam.getPackValue(); auto inputPackTy = inputPackValue.getType().castTo(); @@ -2345,6 +2466,10 @@ class ResultPlanner { /// Valid: NumElements, OuterResult TupleDirect, + /// Take the last direct inner result, which must be a tuple, and + /// split it into its elements. + DestructureDirectInnerTuple, + /// Take the last direct outer result, inject it into an optional /// type, and make that a new direct outer result. /// @@ -2402,6 +2527,14 @@ class ResultPlanner { /// /// Valid: reabstraction info, InnerResult, OuterResult. ReabstractDirectToDirect, + + /// Reabstract the elements of the inner result address (a tuple) + /// into the elements of the given component of the outer result + /// address. + /// + /// Valid: reabstraction info, InnerResultAddr, OuterResultAddr, + /// PackExpansion. + ReabstractTupleIntoPackExpansion, }; Operation(Kind kind) : TheKind(kind) {} @@ -2424,6 +2557,18 @@ class ResultPlanner { SILValue OuterResultAddr; SILResultInfo OuterResult; }; + + union { + struct { + CanPackType OuterFormalPackType; + CanPackType InnerFormalPackType; + unsigned OuterComponentIndex; + unsigned InnerComponentIndex; + } PackExpansion; + }; + + void emitReabstractTupleIntoPackExpansion(SILGenFunction &SGF, + SILLocation loc); }; struct PlanData { @@ -2470,10 +2615,10 @@ class ResultPlanner { .getNumIndirectSILResults()); } - SILValue execute(SILValue innerResult); + SILValue execute(SILValue innerResult, CanSILFunctionType innerFuncTy); private: - void execute(ArrayRef innerDirectResults, + void execute(SmallVectorImpl &innerDirectResults, SmallVectorImpl &outerDirectResults); void executeInnerTuple(SILValue innerElement, SmallVector &innerDirectResults); @@ -2482,70 +2627,136 @@ class ResultPlanner { AbstractionPattern outerOrigType, CanType outerSubstType, PlanData &planData); - void planIntoIndirectResult(AbstractionPattern innerOrigType, + void planParallelExpanded(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData); + void planFromExpanded(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData); + void planIntoExpanded(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData); + void planSingle(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILResultInfo innerResult, + SILResultInfo outerResult, + SILValue optOuterResultAddr); + + void expandVanishingTuple(AbstractionPattern origType, CanType substType, + PlanData &planData, bool forInner, + llvm::function_ref handleSingle, + llvm::function_ref handlePackElement); + + void planIntoIndirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILValue outerResultAddr); + void planExpandedIntoIndirect(AbstractionPattern innerOrigType, + CanTupleType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILValue outerResultAddr); + void planSingleIntoIndirect(AbstractionPattern innerOrigType, CanType innerSubstType, AbstractionPattern outerOrigType, CanType outerSubstType, PlanData &planData, + SILResultInfo innerResult, SILValue outerResultAddr); - void planTupleIntoIndirectResult(AbstractionPattern innerOrigType, - CanTupleType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILValue outerResultAddr); - void planScalarIntoIndirectResult(AbstractionPattern innerOrigType, - CanType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILResultInfo innerResult, - SILValue outerResultAddr); - - void planIntoDirectResult(AbstractionPattern innerOrigType, + + void planIntoDirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILResultInfo outerResult); + void planSingleIntoDirect(AbstractionPattern innerOrigType, CanType innerSubstType, AbstractionPattern outerOrigType, CanType outerSubstType, PlanData &planData, + SILResultInfo innerResult, SILResultInfo outerResult); - void planScalarIntoDirectResult(AbstractionPattern innerOrigType, - CanType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILResultInfo innerResult, - SILResultInfo outerResult); - void planTupleIntoDirectResult(AbstractionPattern innerOrigType, - CanTupleType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILResultInfo outerResult); - - void planFromIndirectResult(AbstractionPattern innerOrigType, - CanType innerSubstType, + void planExpandedIntoDirect(AbstractionPattern innerOrigType, + CanTupleType innerSubstType, AbstractionPattern outerOrigType, CanType outerSubstType, PlanData &planData, - SILValue innerResultAddr); - void planTupleFromIndirectResult(AbstractionPattern innerOrigType, - CanTupleType innerSubstType, - AbstractionPattern outerOrigType, - CanTupleType outerSubstType, - PlanData &planData, - SILValue innerResultAddr); - void planTupleFromDirectResult(AbstractionPattern innerOrigType, - CanTupleType innerSubstType, + SILResultInfo outerResult); + + void planFromIndirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILValue innerResultAddr); + void planFromDirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILResultInfo innerResult); + void planExpandedFromIndirect(AbstractionPattern innerOrigType, + CanTupleType innerSubstType, + AbstractionPattern outerOrigType, + CanTupleType outerSubstType, + PlanData &planData, + SILValue innerResultAddr); + void planExpandedFromDirect(AbstractionPattern innerOrigType, + CanTupleType innerSubstType, + AbstractionPattern outerOrigType, + CanTupleType outerSubstType, + PlanData &planData, + SILResultInfo innerResult); + void planSingleFromIndirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + SILValue innerResultAddr, + SILResultInfo outerResult, + SILValue optOuterResultAddr); + void planIndirectIntoIndirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + SILValue innerResultAddr, + SILValue outerResultAddr); + + void planPackExpansionFromPack(AbstractionPattern innerOrigType, + CanType innerSubstType, AbstractionPattern outerOrigType, - CanTupleType outerSubstType, - PlanData &planData, SILResultInfo innerResult); - void planScalarFromIndirectResult(AbstractionPattern innerOrigType, - CanType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - SILValue innerResultAddr, - SILResultInfo outerResult, - SILValue optOuterResultAddr); + CanType outerSubstType, + CanPackType innerFormalPackType, + SILValue innerPackAddr, + unsigned innerPackComponentIndex, + CanPackType outerFormalPackType, + SILValue outerTupleOrPackAddr, + unsigned outerPackComponentIndex); + void planPackExpansionFromTuple(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + CanPackType innerFormalPackType, + SILValue innerTupleAddr, + unsigned innerComponentIndex, + CanPackType outerFormalPackType, + SILValue outerTupleOrPackAddr, + unsigned outerComponentIndex); /// Claim the next inner result from the plan data. SILResultInfo claimNextInnerResult(PlanData &data) { @@ -2579,6 +2790,80 @@ class ResultPlanner { return temporary; } + SILValue addInnerIndirectPackResultTemporary(PlanData &data, + SILResultInfo innerResult) { + assert(innerResult.isPack()); + assert(SGF.silConv.isSILIndirect(innerResult)); + auto temporary = + SGF.emitTemporaryPackAllocation(Loc, + SGF.getSILType(innerResult, CanSILFunctionType())); + data.InnerIndirectResultAddrs.push_back(temporary); + return temporary; + } + + class InnerPackResultGenerator { + ResultPlanner &planner; + PlanData &data; + public: + InnerPackResultGenerator(ResultPlanner &planner, PlanData &data) + : planner(planner), data(data) {} + + bool isFinished() const { + return data.InnerResults.empty(); + } + ManagedValue claimNext() { + SILResultInfo resultInfo = planner.claimNextInnerResult(data); + return ManagedValue::forLValue( + planner.addInnerIndirectPackResultTemporary(data, resultInfo)); + } + void advance() { + llvm_unreachable("should always be claimed from"); + } + void finish() { + llvm_unreachable("should not be finished directly"); + } + }; + + class OuterPackResultGenerator { + ResultPlanner &planner; + PlanData &data; + public: + OuterPackResultGenerator(ResultPlanner &planner, PlanData &data) + : planner(planner), data(data) {} + + bool isFinished() const { + return data.OuterResults.empty(); + } + ManagedValue claimNext() { + auto resultPair = planner.claimNextOuterResult(data); + assert(resultPair.first.isPack()); + return ManagedValue::forLValue(resultPair.second); + } + void advance() { + llvm_unreachable("should always be claimed from"); + } + void finish() { + llvm_unreachable("should not be finished directly"); + } + }; + + bool hasAbstractionDifference(SILValue resultAddr1, + SILValue resultAddr2) { + return hasAbstractionDifference(resultAddr1->getType(), + resultAddr2->getType()); + } + + bool hasAbstractionDifference(SILType resultType1, + SILType resultType2) { + return resultType1.getASTType() != resultType2.getASTType(); + } + + bool hasAbstractionDifference(SILValue resultAddr, + SILResultInfo resultInfo) { + return resultAddr->getType().getASTType() + != resultInfo.getInterfaceType(); + } + /// Cause the next inner indirect result to be emitted directly into /// the given outer result address. void addInPlace(PlanData &data, SILValue outerResultAddr) { @@ -2624,6 +2909,11 @@ class ResultPlanner { op.OuterResult = outerResult; } + void addDestructureDirectInnerTuple(SILResultInfo innerResult) { + auto &op = addOperation(Operation::DestructureDirectInnerTuple); + op.InnerResult = innerResult; + } + void addInjectOptionalDirect(EnumElementDecl *someDecl, SILResultInfo outerResult) { auto &op = addOperation(Operation::InjectOptionalDirect); @@ -2697,6 +2987,29 @@ class ResultPlanner { op.OuterOrigType = outerOrigType; op.OuterSubstType = outerSubstType; } + + void addReabstractTupleIntoPackExpansion(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + CanPackType innerFormalPackType, + SILValue innerResultAddr, + unsigned innerComponentIndex, + CanPackType outerFormalPackType, + SILValue outerResultAddr, + unsigned outerComponentIndex) { + auto &op = addOperation(Operation::ReabstractTupleIntoPackExpansion); + op.InnerResultAddr = innerResultAddr; + op.OuterResultAddr = outerResultAddr; + op.InnerOrigType = innerOrigType; + op.InnerSubstType = innerSubstType; + op.OuterOrigType = outerOrigType; + op.OuterSubstType = outerSubstType; + op.PackExpansion.InnerFormalPackType = innerFormalPackType; + op.PackExpansion.InnerComponentIndex = innerComponentIndex; + op.PackExpansion.OuterFormalPackType = outerFormalPackType; + op.PackExpansion.OuterComponentIndex = outerComponentIndex; + } }; } // end anonymous namespace @@ -2716,125 +3029,548 @@ void ResultPlanner::plan(AbstractionPattern innerOrigType, cast(innerSubstType)->getNumElements() == cast(outerSubstType)->getNumElements()); - // If the inner abstraction pattern is a tuple, that result will be expanded. - if (innerOrigType.isTuple()) { - auto innerSubstTupleType = cast(innerSubstType); - - // If the outer abstraction pattern is also a tuple, that result will also - // be expanded, in parallel with the inner pattern. - if (outerOrigType.isTuple()) { - auto outerSubstTupleType = cast(outerSubstType); - assert(innerSubstTupleType->getNumElements() - == outerSubstTupleType->getNumElements()); - - // Otherwise, recursively descend into the tuples. - for (auto eltIndex : indices(innerSubstTupleType.getElementTypes())) { - plan(innerOrigType.getTupleElementType(eltIndex), - innerSubstTupleType.getElementType(eltIndex), - outerOrigType.getTupleElementType(eltIndex), - outerSubstTupleType.getElementType(eltIndex), + // Tuple results in the abstraction pattern are expanded. + // If we have a vanishing tuple on one side or the other, + // we should look through that structure immediately and recurse. + // Otherwise, if we have non-vanishing tuple structure on both sides, + // we need to walk it in parallel. + // Otherwise, we should only have non-vanishing tuple structure on + // the input side, and the output side must be Any or optional. + + // If the outer abstraction pattern is a vanishing tuple, look + // through that and recurse. + bool outerIsExpanded = outerOrigType.isTuple(); + if (outerIsExpanded && outerOrigType.doesTupleVanish()) { + expandVanishingTuple(outerOrigType, outerSubstType, planData, /*inner*/ false, + [&](AbstractionPattern outerOrigEltType, CanType outerSubstEltType) { + plan(innerOrigType, innerSubstType, + outerOrigEltType, outerSubstEltType, planData); + }, [&](AbstractionPattern outerOrigEltType, CanType outerSubstEltType, + SILValue outerResultAddr) { + planIntoIndirect(innerOrigType, innerSubstType, + outerOrigEltType, outerSubstEltType, + planData, outerResultAddr); + }); + return; + } + + // If the inner abstraction pattern is a vanishing tuple, look + // through that and recurse. + bool innerIsExpanded = innerOrigType.isTuple(); + if (innerIsExpanded && innerOrigType.doesTupleVanish()) { + expandVanishingTuple(innerOrigType, innerSubstType, planData, /*inner*/ true, + [&](AbstractionPattern innerOrigEltType, CanType innerSubstEltType) { + plan(innerOrigEltType, innerSubstEltType, + outerOrigType, outerSubstType, planData); + }, [&](AbstractionPattern innerOrigEltType, CanType innerSubstEltType, + SILValue innerResultAddr) { + planFromIndirect(innerOrigEltType, innerSubstEltType, + outerOrigType, outerSubstType, + planData, innerResultAddr); + }); + return; + } + + if (innerIsExpanded && outerIsExpanded) { + planParallelExpanded(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, planData); + } else if (outerIsExpanded) { + planIntoExpanded(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, planData); + } else if (innerIsExpanded) { + planFromExpanded(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, planData); + } else { + SILResultInfo innerResult = claimNextInnerResult(planData); + auto outerResultPair = claimNextOuterResult(planData); + SILResultInfo outerResult = outerResultPair.first; + SILValue outerResultAddr = outerResultPair.second; + planSingle(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + planData, innerResult, outerResult, outerResultAddr); + } +} + +void ResultPlanner::expandVanishingTuple(AbstractionPattern origType, + CanType substType, + PlanData &planData, + bool forInner, + llvm::function_ref handleSingle, + llvm::function_ref handlePackElement) { + + bool foundSurvivor = false; + + Optional innerPacks; + Optional outerPacks; + Optional> packInputs; + if (forInner) { + innerPacks.emplace(*this, planData); + packInputs.emplace(*innerPacks); + } else { + outerPacks.emplace(*this, planData); + packInputs.emplace(*outerPacks); + } + + ExpandedTupleInputGenerator elt(SGF.getASTContext(), *packInputs, + origType, substType); + for (; !elt.isFinished(); elt.advance()) { + assert(!foundSurvivor); + foundSurvivor = true; + + if (!elt.isOrigPackExpansion()) { + handleSingle(elt.getOrigType(), elt.getSubstType()); + } else { + assert(elt.getPackComponentIndex() == 0); + assert(!isa(elt.getSubstType())); + SILValue eltAddr = [&] { + if (forInner) + return elt.createPackComponentTemporary(SGF, Loc); + return elt.projectPackComponent(SGF, Loc).getLValueAddress(); + }(); + handlePackElement(elt.getOrigType().getPackExpansionPatternType(), + substType, eltAddr); + } + } + elt.finish(); + + assert(foundSurvivor && "vanishing tuple had no surviving element?"); +} + +void ResultPlanner::planParallelExpanded(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData) { + assert(innerOrigType.isTuple()); + assert(!innerOrigType.doesTupleVanish()); + assert(outerOrigType.isTuple()); + assert(!outerOrigType.doesTupleVanish()); + + auto &ctx = SGF.getASTContext(); + InnerPackResultGenerator innerPacks(*this, planData); + OuterPackResultGenerator outerPacks(*this, planData); + ExpandedTupleInputGenerator innerElt(ctx, innerPacks, + innerOrigType, innerSubstType); + ExpandedTupleInputGenerator outerElt(ctx, outerPacks, + outerOrigType, outerSubstType); + + for (; !innerElt.isFinished(); innerElt.advance(), outerElt.advance()) { + assert(!outerElt.isFinished() && "elements not parallel"); + assert(outerElt.isSubstPackExpansion() == innerElt.isSubstPackExpansion()); + + // If this substituted component is a pack expansion, handle that + // immediately. + if (auto outerSubstExpansionType = + dyn_cast(outerElt.getSubstType())) { + planPackExpansionFromPack(innerElt.getOrigType(), + cast(innerElt.getSubstType()).getPatternType(), + outerElt.getOrigType(), + outerSubstExpansionType.getPatternType(), + innerElt.getFormalPackType(), + innerElt.getPackValue().getLValueAddress(), + innerElt.getPackComponentIndex(), + outerElt.getFormalPackType(), + outerElt.getPackValue().getLValueAddress(), + outerElt.getPackComponentIndex()); + continue; + } + + AbstractionPattern outerOrigEltType = outerElt.getOrigType(); + AbstractionPattern innerOrigEltType = innerElt.getOrigType(); + CanType outerSubstEltType = outerElt.getSubstType(); + CanType innerSubstEltType = innerElt.getSubstType(); + + if (outerElt.isOrigPackExpansion()) { + auto outerEltAddr = + outerElt.projectPackComponent(SGF, Loc).getLValueAddress(); + if (innerElt.isOrigPackExpansion()) { + auto innerEltAddr = + innerElt.createPackComponentTemporary(SGF, Loc); + planIndirectIntoIndirect(innerOrigEltType, innerSubstEltType, + outerOrigEltType, outerSubstEltType, + innerEltAddr, outerEltAddr); + } else { + planIntoIndirect(innerOrigEltType, innerSubstEltType, + outerOrigEltType, outerSubstEltType, + planData, outerEltAddr); + } + } else { + if (innerElt.isOrigPackExpansion()) { + auto innerEltAddr = innerElt.createPackComponentTemporary(SGF, Loc); + planFromIndirect(innerOrigEltType, innerSubstEltType, + outerOrigEltType, outerSubstEltType, + planData, innerEltAddr); + } else { + plan(innerOrigEltType, innerSubstEltType, + outerOrigEltType, outerSubstEltType, planData); } - return; } + } + assert(outerElt.isFinished() && "elements not parallel"); + outerElt.finish(); + innerElt.finish(); +} - // Otherwise, the next outer result must be either opaque or optional. - // In either case, it corresponds to a single result. - auto outerResult = claimNextOuterResult(planData); +static SILType getTupleOrPackElementType(SILType valueType, + unsigned componentIndex) { + if (auto packType = valueType.getAs()) { + return packType->getSILElementType(componentIndex); + } else { + return valueType.getTupleElementType(componentIndex); + } +} - // Base the plan on whether the single result is direct or indirect. - if (SGF.silConv.isSILIndirect(outerResult.first)) { - assert(outerResult.second); - planTupleIntoIndirectResult(innerOrigType, innerSubstTupleType, - outerOrigType, outerSubstType, - planData, outerResult.second); +static SILValue emitTupleOrPackElementAddr(SILGenFunction &SGF, + SILLocation loc, + SILValue packIndex, + SILValue tupleOrPackAddr, + SILType eltTy) { + if (tupleOrPackAddr->getType().is()) { + return SGF.B.createPackElementGet(loc, packIndex, tupleOrPackAddr, eltTy); + } else { + return SGF.B.createTuplePackElementAddr(loc, packIndex, tupleOrPackAddr, + eltTy); + } +} + +/// We have a pack expansion in a substituted result type, and the inner +/// result type is expanded. Emit a pack loop to set up the indirect +/// result pack for the callee, then add an operation to reabstract the +/// result back if necessary. +void ResultPlanner::planPackExpansionFromPack( + AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + CanPackType innerFormalPackType, + SILValue innerPackAddr, + unsigned innerPackComponentIndex, + CanPackType outerFormalPackType, + SILValue outerTupleOrPackAddr, + unsigned outerComponentIndex) { + // The orig and subst types are the pattern types, not the expansion types. + + SILType innerPackExpansionTy = + innerPackAddr->getType().getPackElementType(innerPackComponentIndex); + SILType outerPackExpansionTy = + getTupleOrPackElementType(outerTupleOrPackAddr->getType(), + outerComponentIndex); + + SILType innerEltTy, outerEltTy; + auto openedEnv = SGF.createOpenedElementValueEnvironment( + { innerPackExpansionTy, outerPackExpansionTy }, + { &innerEltTy, &outerEltTy }); + + // If the pack elements need reabstraction, we need to collect results + // into the elements of a temporary tuple and then reabstract after the + // call. + SILValue innerTemporaryAddr; + bool reabstract = hasAbstractionDifference(innerEltTy, outerEltTy); + if (reabstract) { + auto innerTemporaryTy = + SILType::getPrimitiveObjectType(CanType( + TupleType::get({innerPackExpansionTy.getASTType()}, + SGF.getASTContext()))); + innerTemporaryAddr = SGF.emitTemporaryAllocation(Loc, innerTemporaryTy); + } + + // Perform a pack loop to set the element addresses for this pack + // expansion in the inner pack. + SGF.emitDynamicPackLoop(Loc, innerFormalPackType, innerPackComponentIndex, + openedEnv, + [&](SILValue indexWithinComponent, + SILValue packExpansionIndex, + SILValue innerPackIndex) { + SILValue innerEltAddr; + if (reabstract) { + innerEltAddr = SGF.B.createTuplePackElementAddr(Loc, packExpansionIndex, + innerTemporaryAddr, + innerEltTy); } else { - planTupleIntoDirectResult(innerOrigType, innerSubstTupleType, - outerOrigType, outerSubstType, - planData, outerResult.first); + SILValue outerPackIndex = packExpansionIndex; + if (outerFormalPackType->getNumElements() != 1) { + outerPackIndex = + SGF.B.createPackPackIndex(Loc, outerComponentIndex, + outerPackIndex, outerFormalPackType); + } + + // Since we're not reabstracting, we can use the outer address + // directly as the inner address. + innerEltAddr = emitTupleOrPackElementAddr(SGF, Loc, outerPackIndex, + outerTupleOrPackAddr, + outerEltTy); } - return; + + SGF.B.createPackElementSet(Loc, innerEltAddr, innerPackIndex, + innerPackAddr); + }); + + if (reabstract) { + auto innerFormalPackType = + innerTemporaryAddr->getType().castTo().getInducedPackType(); + unsigned innerComponentIndex = 0; + + addReabstractTupleIntoPackExpansion(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + innerFormalPackType, + innerTemporaryAddr, + innerComponentIndex, + outerFormalPackType, + outerTupleOrPackAddr, + outerComponentIndex); } +} - // Otherwise, the inner pattern is a scalar; claim the next inner result. - SILResultInfo innerResult = claimNextInnerResult(planData); +/// We have a pack expansion in a substituted result type, and the inner +/// result type is not expanded. Add an operation to move the result +/// into the outer pack or tuple, reabstracting if necessary. +void ResultPlanner::planPackExpansionFromTuple( + AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + CanPackType innerInducedPackType, + SILValue innerTupleAddr, + unsigned innerComponentIndex, + CanPackType outerFormalPackType, + SILValue outerPackOrTupleAddr, + unsigned outerComponentIndex) { + // The orig and subst types are the pattern types, not the expansion types. + addReabstractTupleIntoPackExpansion(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + innerInducedPackType, + innerTupleAddr, + innerComponentIndex, + outerFormalPackType, + outerPackOrTupleAddr, + outerComponentIndex); +} - assert((!outerOrigType.isTuple() || innerResult.isFormalIndirect()) && - "outer pattern is a tuple, inner pattern is not, but inner result is " - "not indirect?"); +void ResultPlanner::Operation::emitReabstractTupleIntoPackExpansion( + SILGenFunction &SGF, SILLocation loc) { + SILValue innerTupleAddr = InnerResultAddr; + SILValue outerTupleOrPackAddr = OuterResultAddr; + unsigned innerComponentIndex = PackExpansion.InnerComponentIndex; + unsigned outerComponentIndex = PackExpansion.OuterComponentIndex; + + SILType innerPackExpansionTy = + innerTupleAddr->getType().getTupleElementType(innerComponentIndex); + SILType outerPackExpansionTy = + getTupleOrPackElementType(outerTupleOrPackAddr->getType(), + outerComponentIndex); + + SILType innerEltTy, outerEltTy; + auto openedEnv = SGF.createOpenedElementValueEnvironment( + { innerPackExpansionTy, outerPackExpansionTy }, + { &innerEltTy, &outerEltTy }); + + auto innerFormalPackType = PackExpansion.InnerFormalPackType; + auto outerFormalPackType = PackExpansion.OuterFormalPackType; + + SGF.emitDynamicPackLoop(loc, innerFormalPackType, innerComponentIndex, + openedEnv, + [&](SILValue indexWithinComponent, + SILValue packExpansionIndex, + SILValue innerPackIndex) { + // Construct a managed value for the inner element, loading it + // if appropriate. We assume the result is owned. + auto innerEltAddr = + SGF.B.createTuplePackElementAddr(loc, innerPackIndex, innerTupleAddr, + innerEltTy); + auto innerEltValue = + SGF.emitManagedBufferWithCleanup(innerEltAddr); + innerEltValue = SGF.B.createLoadIfLoadable(loc, innerEltValue); + + // Project the address of the outer element. + auto outerPackIndex = packExpansionIndex; + if (outerFormalPackType->getNumElements() != 1) { + outerPackIndex = SGF.B.createPackPackIndex(loc, outerComponentIndex, + outerPackIndex, + outerFormalPackType); + } + auto outerEltAddr = + emitTupleOrPackElementAddr(SGF, loc, outerPackIndex, + outerTupleOrPackAddr, outerEltTy); + + // Set up to perform the reabstraction into the outer result. + TemporaryInitialization outerEltInit(outerEltAddr, + CleanupHandle::invalid()); + auto outerResultCtxt = SGFContext(&outerEltInit); + + CanType innerSubstType = InnerSubstType; + CanType outerSubstType = OuterSubstType; + if (openedEnv) { + innerSubstType = + openedEnv->mapContextualPackTypeIntoElementContext(innerSubstType); + outerSubstType = + openedEnv->mapContextualPackTypeIntoElementContext(outerSubstType); + } - // If the inner result is a tuple, we need to expand from a temporary. - if (innerResult.isFormalIndirect() && outerOrigType.isTuple()) { - if (SGF.silConv.isSILIndirect(innerResult)) { - SILValue innerResultAddr = - addInnerIndirectResultTemporary(planData, innerResult); - planTupleFromIndirectResult( - innerOrigType, cast(innerSubstType), outerOrigType, - cast(outerSubstType), planData, innerResultAddr); - } else { - assert(!SGF.silConv.useLoweredAddresses() && - "Formal Indirect Results that are not SIL Indirect are only " - "allowed in opaque values mode"); - planTupleFromDirectResult(innerOrigType, cast(innerSubstType), - outerOrigType, cast(outerSubstType), - planData, innerResult); + // Reabstract. + auto outerEltValue = + SGF.emitTransformedValue(loc, innerEltValue, + InnerOrigType, innerSubstType, + OuterOrigType, outerSubstType, + outerEltTy, outerResultCtxt); + + // Force the value into the outer result address if necessary. + if (!outerEltValue.isInContext()) { + outerEltValue.forwardInto(SGF, loc, outerEltAddr); } - return; + }); +} + +void ResultPlanner::planIntoExpanded(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData) { + assert(outerOrigType.isTuple()); + assert(!outerOrigType.doesTupleVanish()); + assert(!innerOrigType.isTuple()); + + // We know that the outer tuple is not vanishing (because the top-level + // plan function filters that out), so the outer subst type must be a + // tuple. The inner subst type must also be a tuple because only tuples + // convert to tuples. + auto outerSubstTupleType = cast(outerSubstType); + auto innerSubstTupleType = cast(innerSubstType); + + // The next inner result is not expanded, so there's a single result. + SILResultInfo innerResult = claimNextInnerResult(planData); + + if (SGF.silConv.isSILIndirect(innerResult)) { + SILValue innerResultAddr = + addInnerIndirectResultTemporary(planData, innerResult); + planExpandedFromIndirect(innerOrigType, innerSubstTupleType, + outerOrigType, outerSubstTupleType, + planData, innerResultAddr); + } else { + assert(!SGF.silConv.useLoweredAddresses() && + "Formal Indirect Results that are not SIL Indirect are only " + "allowed in opaque values mode"); + planExpandedFromDirect(innerOrigType, innerSubstTupleType, + outerOrigType, outerSubstTupleType, + planData, innerResult); } +} - // Otherwise, the outer pattern is a scalar; claim the next outer result. - auto outerResult = claimNextOuterResult(planData); +void ResultPlanner::planFromExpanded(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData) { + assert(innerOrigType.isTuple()); + assert(!innerOrigType.doesTupleVanish()); + assert(!outerOrigType.isTuple()); + + // We know that the inner tuple is not vanishing (because the top-level + // plan function filters that out), so the inner subst type must be a + // tuple. The outer subst type might not be a tuple if it's e.g. Any. + auto innerSubstTupleType = cast(innerSubstType); + + // The next outer result is not expanded, so there's a single result. + auto outerResultPair = claimNextOuterResult(planData); + SILResultInfo outerResult = outerResultPair.first; + SILValue outerResultAddr = outerResultPair.second; + + // Base the plan on whether the single result is direct or indirect. + if (SGF.silConv.isSILIndirect(outerResult)) { + assert(outerResultAddr); + planExpandedIntoIndirect(innerOrigType, innerSubstTupleType, + outerOrigType, outerSubstType, + planData, outerResultAddr); + } else { + assert(!outerResultAddr); + planExpandedIntoDirect(innerOrigType, innerSubstTupleType, + outerOrigType, outerSubstType, + planData, outerResult); + } +} +void ResultPlanner::planSingle(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILResultInfo innerResult, + SILResultInfo outerResult, + SILValue optOuterResultAddr) { // If the outer result is indirect, plan to emit into that. - if (SGF.silConv.isSILIndirect(outerResult.first)) { - assert(outerResult.second); - planScalarIntoIndirectResult(innerOrigType, innerSubstType, - outerOrigType, outerSubstType, - planData, innerResult, outerResult.second); + if (SGF.silConv.isSILIndirect(outerResult)) { + assert(optOuterResultAddr); + planSingleIntoIndirect(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + planData, innerResult, optOuterResultAddr); } else { - planScalarIntoDirectResult(innerOrigType, innerSubstType, - outerOrigType, outerSubstType, - planData, innerResult, outerResult.first); + assert(!optOuterResultAddr); + planSingleIntoDirect(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + planData, innerResult, outerResult); } } /// Plan the emission of a call result into an outer result address. -void ResultPlanner::planIntoIndirectResult(AbstractionPattern innerOrigType, - CanType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILValue outerResultAddr) { - // outerOrigType can be a tuple if we're also injecting into an optional. - - // If the inner pattern is a tuple, expand it. - if (innerOrigType.isTuple()) { - planTupleIntoIndirectResult(innerOrigType, cast(innerSubstType), - outerOrigType, outerSubstType, - planData, outerResultAddr); +void ResultPlanner::planIntoIndirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILValue outerResultAddr) { + // outerOrigType can be a tuple if we're e.g. injecting into an optional; + // we just know we're not expanding it. - // Otherwise, it's scalar. - } else { + // If the inner pattern is not a tuple, claim the next inner result + // and plan from that. + if (!innerOrigType.isTuple()) { // Claim the next inner result. SILResultInfo innerResult = claimNextInnerResult(planData); - planScalarIntoIndirectResult(innerOrigType, innerSubstType, - outerOrigType, outerSubstType, - planData, innerResult, outerResultAddr); + planSingleIntoIndirect(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + planData, innerResult, outerResultAddr); + return; + } + + // If the inner pattern is not vanishing, expand the tuple. + if (!innerOrigType.doesTupleVanish()) { + planExpandedIntoIndirect(innerOrigType, cast(innerSubstType), + outerOrigType, outerSubstType, + planData, outerResultAddr); + return; } + + // Otherwise, we have a vanishing tuple. Expand it, find the surviving + // element, and recurse. + expandVanishingTuple(innerOrigType, innerSubstType, planData, /*inner*/ true, + [&](AbstractionPattern innerOrigEltType, CanType innerSubstEltType) { + planIntoIndirect(innerOrigEltType, innerSubstEltType, + outerOrigType, outerSubstType, planData, + outerResultAddr); + }, [&](AbstractionPattern innerOrigEltType, + CanType innerSubstEltType, SILValue innerResultAddr) { + planIndirectIntoIndirect(innerOrigEltType, innerSubstEltType, + outerOrigType, outerSubstType, + innerResultAddr, outerResultAddr); + }); } /// Plan the emission of a call result into an outer result address, /// given that the inner abstraction pattern is a tuple. void -ResultPlanner::planTupleIntoIndirectResult(AbstractionPattern innerOrigType, - CanTupleType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILValue outerResultAddr) { +ResultPlanner::planExpandedIntoIndirect(AbstractionPattern innerOrigType, + CanTupleType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILValue outerResultAddr) { assert(innerOrigType.isTuple()); + assert(!innerOrigType.doesTupleVanish()); // outerOrigType can be a tuple if we're doing something like // injecting into an optional tuple. @@ -2855,7 +3591,7 @@ ResultPlanner::planTupleIntoIndirectResult(AbstractionPattern innerOrigType, outerObjectType); // Emit into that address. - planTupleIntoIndirectResult( + planExpandedIntoIndirect( innerOrigType, innerSubstType, outerOrigType.getOptionalObjectType(), outerSubstObjectType, planData, outerObjectResultAddr); @@ -2874,66 +3610,113 @@ ResultPlanner::planTupleIntoIndirectResult(AbstractionPattern innerOrigType, /*conformances=*/{}); // Emit into that address. - planTupleIntoIndirectResult(innerOrigType, innerSubstType, - innerOrigType, innerSubstType, - planData, outerConcreteResultAddr); + planExpandedIntoIndirect(innerOrigType, innerSubstType, + innerOrigType, innerSubstType, + planData, outerConcreteResultAddr); return; } - assert(innerSubstType->getNumElements() - == outerSubstTupleType->getNumElements()); + // Otherwise, we're doing a tuple-to-tuple conversion, which is a + // parallel walk. - for (auto eltIndex : indices(innerSubstType.getElementTypes())) { - // Project the address of the element. - SILValue outerEltResultAddr = - SGF.B.createTupleElementAddr(Loc, outerResultAddr, eltIndex); + auto &ctx = SGF.getASTContext(); + InnerPackResultGenerator innerPacks(*this, planData); + ExpandedTupleInputGenerator innerElt(ctx, innerPacks, + innerOrigType, innerSubstType); + TupleElementAddressGenerator outerElt(ctx, + ManagedValue::forLValue(outerResultAddr), + outerOrigType, + outerSubstTupleType); - // Plan to emit into that location. - planIntoIndirectResult(innerOrigType.getTupleElementType(eltIndex), - innerSubstType.getElementType(eltIndex), - outerOrigType.getTupleElementType(eltIndex), - outerSubstTupleType.getElementType(eltIndex), - planData, outerEltResultAddr); - } + for (; !outerElt.isFinished(); outerElt.advance(), innerElt.advance()) { + assert(!innerElt.isFinished()); + assert(innerElt.isSubstPackExpansion() == outerElt.isSubstPackExpansion()); + + SILValue outerEltAddr = + outerElt.projectElementAddress(SGF, Loc).getLValueAddress(); + + if (!innerElt.isOrigPackExpansion()) { + planIntoIndirect(innerElt.getOrigType(), innerElt.getSubstType(), + outerElt.getOrigType(), outerElt.getSubstType(), + planData, outerEltAddr); + continue; + } + + if (!innerElt.isSubstPackExpansion()) { + SILValue innerEltAddr = innerElt.createPackComponentTemporary(SGF, Loc); + planIndirectIntoIndirect(innerElt.getOrigType(), innerElt.getSubstType(), + outerElt.getOrigType(), outerElt.getSubstType(), + innerEltAddr, outerEltAddr); + continue; + } + + planPackExpansionFromPack(innerElt.getOrigType(), + cast(innerElt.getSubstType()).getPatternType(), + outerElt.getOrigType(), + cast(outerElt.getSubstType()).getPatternType(), + innerElt.getFormalPackType(), + innerElt.getPackValue().getLValueAddress(), + innerElt.getPackComponentIndex(), + outerElt.getInducedPackType(), + outerEltAddr, + outerElt.getSubstElementIndex()); + } + innerElt.finish(); + outerElt.finish(); } /// Plan the emission of a call result as a single outer direct result. -void -ResultPlanner::planIntoDirectResult(AbstractionPattern innerOrigType, - CanType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILResultInfo outerResult) { +void ResultPlanner::planIntoDirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILResultInfo outerResult) { assert(!outerOrigType.isTuple() || !SGF.silConv.useLoweredAddresses()); - // If the inner pattern is a tuple, expand it. - if (innerOrigType.isTuple()) { - planTupleIntoDirectResult(innerOrigType, cast(innerSubstType), - outerOrigType, outerSubstType, - planData, outerResult); - - // Otherwise, it's scalar. - } else { - // Claim the next inner result. + // If the inner pattern is scalar, claim the next inner result. + if (!innerOrigType.isTuple()) { SILResultInfo innerResult = claimNextInnerResult(planData); - planScalarIntoDirectResult(innerOrigType, innerSubstType, - outerOrigType, outerSubstType, - planData, innerResult, outerResult); + planSingleIntoDirect(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + planData, innerResult, outerResult); + return; + } + + // If the inner pattern doesn't vanish, expand it. + if (!innerOrigType.doesTupleVanish()) { + planExpandedIntoDirect(innerOrigType, cast(innerSubstType), + outerOrigType, outerSubstType, + planData, outerResult); + return; } + + // Otherwise, the inner tuple vanishes. Expand it, find the surviving + // element, and recurse. + expandVanishingTuple(innerOrigType, innerSubstType, planData, /*inner*/ true, + [&](AbstractionPattern innerOrigEltType, CanType innerSubstEltType) { + planIntoDirect(innerOrigEltType, innerSubstEltType, + outerOrigType, outerSubstType, planData, + outerResult); + }, [&](AbstractionPattern innerOrigEltType, + CanType innerSubstEltType, SILValue innerResultAddr) { + planSingleFromIndirect(innerOrigEltType, innerSubstEltType, + outerOrigType, outerSubstType, + innerResultAddr, outerResult, SILValue()); + }); } /// Plan the emission of a call result as a single outer direct result, /// given that the inner abstraction pattern is a tuple. -void -ResultPlanner::planTupleIntoDirectResult(AbstractionPattern innerOrigType, - CanTupleType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILResultInfo outerResult) { +void ResultPlanner::planExpandedIntoDirect(AbstractionPattern innerOrigType, + CanTupleType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILResultInfo outerResult) { assert(innerOrigType.isTuple()); + assert(!innerOrigType.doesTupleVanish()); auto outerSubstTupleType = dyn_cast(outerSubstType); @@ -2951,7 +3734,7 @@ ResultPlanner::planTupleIntoDirectResult(AbstractionPattern innerOrigType, outerResult.getConvention()); // Plan to leave the tuple elements as a single direct outer result. - planTupleIntoDirectResult( + planExpandedIntoDirect( innerOrigType, innerSubstType, outerOrigType.getOptionalObjectType(), outerSubstObjectType, planData, outerObjectResult); @@ -2972,9 +3755,9 @@ ResultPlanner::planTupleIntoDirectResult(AbstractionPattern innerOrigType, Loc, outerResultAddr, innerSubstType, SGF.getLoweredType(opaque, innerSubstType), /*conformances=*/{}); - planTupleIntoIndirectResult(innerOrigType, innerSubstType, innerOrigType, - innerSubstType, planData, - outerConcreteResultAddr); + planExpandedIntoIndirect(innerOrigType, innerSubstType, + innerOrigType, innerSubstType, + planData, outerConcreteResultAddr); addReabstractIndirectToDirect(innerOrigType, innerSubstType, outerOrigType, outerSubstType, @@ -2983,38 +3766,73 @@ ResultPlanner::planTupleIntoDirectResult(AbstractionPattern innerOrigType, } } - // Otherwise, the outer type is a tuple. + // Otherwise, the outer type is a tuple with parallel structure + // to the inner type. assert(innerSubstType->getNumElements() == outerSubstTupleType->getNumElements()); - // Create direct outer results for each of the elements. - for (auto eltIndex : indices(innerSubstType.getElementTypes())) { - auto outerEltType = - SGF.getSILType(outerResult, CanSILFunctionType()) - .getTupleElementType(eltIndex); - SILResultInfo outerEltResult(outerEltType.getASTType(), - outerResult.getConvention()); + auto outerTupleTy = SGF.getSILType(outerResult, CanSILFunctionType()); - planIntoDirectResult(innerOrigType.getTupleElementType(eltIndex), - innerSubstType.getElementType(eltIndex), - outerOrigType.getTupleElementType(eltIndex), - outerSubstTupleType.getElementType(eltIndex), - planData, outerEltResult); + // If the substituted tuples contain pack expansions, we need to + // use the indirect path for the tuple and then add a load operation, + // because we can't do pack loops on scalar tuples in SIL. + if (innerSubstType->containsPackExpansionType()) { + auto temporary = + SGF.emitTemporaryAllocation(Loc, outerTupleTy.getObjectType()); + planExpandedIntoIndirect(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + planData, temporary); + addIndirectToDirect(temporary, outerResult); + return; } - // Bind them together into a single tuple. + // Expand the inner tuple, producing direct outer results for each + // of the elements. + InnerPackResultGenerator innerPacks(*this, planData); + ExpandedTupleInputGenerator innerElt(SGF.getASTContext(), innerPacks, + innerOrigType, innerSubstType); + outerOrigType.forEachExpandedTupleElement(outerSubstType, + [&](AbstractionPattern outerOrigEltType, + CanType outerSubstEltType, + const TupleTypeElt &outerOrigTupleElt) { + assert(!innerElt.isFinished()); + auto eltIndex = innerElt.getSubstElementIndex(); + auto outerEltTy = outerTupleTy.getTupleElementType(eltIndex); + SILResultInfo outerEltResult(outerEltTy.getASTType(), + outerResult.getConvention()); + + // If the inner element is part of an orig pack expansion, it's + // indirect; create a temporary for it and reabstract that back to + // a direct result. + if (innerElt.isOrigPackExpansion()) { + auto innerEltAddr = innerElt.createPackComponentTemporary(SGF, Loc); + planSingleFromIndirect(innerElt.getOrigType(), innerElt.getSubstType(), + outerOrigEltType, outerSubstEltType, + innerEltAddr, outerEltResult, SILValue()); + + // Otherwise, recurse normally. + } else { + planIntoDirect(innerElt.getOrigType(), innerElt.getSubstType(), + outerOrigEltType, outerSubstEltType, + planData, outerEltResult); + } + innerElt.advance(); + }); + innerElt.finish(); + + // Bind those direct results together into a single tuple. addTupleDirect(innerSubstType->getNumElements(), outerResult); } /// Plan the emission of a call result as a single outer direct result, /// given that the inner abstraction pattern is not a tuple. -void ResultPlanner::planScalarIntoDirectResult(AbstractionPattern innerOrigType, - CanType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILResultInfo innerResult, - SILResultInfo outerResult) { +void ResultPlanner::planSingleIntoDirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILResultInfo innerResult, + SILResultInfo outerResult) { assert(!innerOrigType.isTuple()); assert(!outerOrigType.isTuple()); @@ -3022,9 +3840,9 @@ void ResultPlanner::planScalarIntoDirectResult(AbstractionPattern innerOrigType, if (SGF.silConv.isSILIndirect(innerResult)) { SILValue innerResultAddr = addInnerIndirectResultTemporary(planData, innerResult); - planScalarFromIndirectResult(innerOrigType, innerSubstType, - outerOrigType, outerSubstType, - innerResultAddr, outerResult, SILValue()); + planSingleFromIndirect(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + innerResultAddr, outerResult, SILValue()); return; } @@ -3046,24 +3864,22 @@ void ResultPlanner::planScalarIntoDirectResult(AbstractionPattern innerOrigType, /// Plan the emission of a call result into an outer result address, /// given that the inner abstraction pattern is not a tuple. void -ResultPlanner::planScalarIntoIndirectResult(AbstractionPattern innerOrigType, - CanType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILResultInfo innerResult, - SILValue outerResultAddr) { +ResultPlanner::planSingleIntoIndirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILResultInfo innerResult, + SILValue outerResultAddr) { assert(!innerOrigType.isTuple()); - assert(!outerOrigType.isTuple()); - - bool hasAbstractionDifference = - (innerResult.getInterfaceType() != outerResultAddr->getType().getASTType()); + // outerOrigType can be a tuple pattern; we just know it's not expanded + // in this position. // If the inner result is indirect, we need some memory to emit it into. if (SGF.silConv.isSILIndirect(innerResult)) { // If there's no abstraction difference, that can just be // in-place into the outer result address. - if (!hasAbstractionDifference) { + if (!hasAbstractionDifference(outerResultAddr, innerResult)) { addInPlace(planData, outerResultAddr); // Otherwise, we'll need a temporary. @@ -3078,7 +3894,7 @@ ResultPlanner::planScalarIntoIndirectResult(AbstractionPattern innerOrigType, // Otherwise, the inner result is direct. } else { // If there's no abstraction difference, we just need to store. - if (!hasAbstractionDifference) { + if (!hasAbstractionDifference(outerResultAddr, innerResult)) { addDirectToIndirect(innerResult, outerResultAddr); // Otherwise, we need to reabstract and store. @@ -3091,126 +3907,238 @@ ResultPlanner::planScalarIntoIndirectResult(AbstractionPattern innerOrigType, } /// Plan the emission of a call result from an inner result address. -void ResultPlanner::planFromIndirectResult(AbstractionPattern innerOrigType, - CanType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - PlanData &planData, - SILValue innerResultAddr) { +void ResultPlanner::planFromIndirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILValue innerResultAddr) { assert(!innerOrigType.isTuple()); - if (outerOrigType.isTuple()) { - planTupleFromIndirectResult(innerOrigType, cast(innerSubstType), - outerOrigType, cast(outerSubstType), - planData, innerResultAddr); - } else { + // If the outer pattern is scalar, claim the next outer result. + if (!outerOrigType.isTuple()) { auto outerResult = claimNextOuterResult(planData); - planScalarFromIndirectResult(innerOrigType, innerSubstType, - outerOrigType, outerSubstType, - innerResultAddr, - outerResult.first, outerResult.second); + planSingleFromIndirect(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + innerResultAddr, + outerResult.first, outerResult.second); + return; } + + // If the outer pattern is not a vanishing tuple, emit indirect + // into it. + if (!outerOrigType.doesTupleVanish()) { + planExpandedFromIndirect(innerOrigType, cast(innerSubstType), + outerOrigType, cast(outerSubstType), + planData, innerResultAddr); + return; + } + + // Otherwise, the outer pattern is a vanishing tuple. Expand it, + // find the surviving element, and recurse. + expandVanishingTuple(outerOrigType, outerSubstType, planData, /*inner*/ false, + [&](AbstractionPattern outerOrigEltType, CanType outerSubstEltType) { + planFromIndirect(innerOrigType, innerSubstType, + outerOrigEltType, outerSubstEltType, + planData, innerResultAddr); + }, [&](AbstractionPattern outerOrigEltType, CanType outerSubstEltType, + SILValue outerResultAddr) { + planIndirectIntoIndirect(innerOrigType, innerSubstType, + outerOrigEltType, outerSubstEltType, + innerResultAddr, outerResultAddr); + }); } /// Plan the emission of a call result from an inner result address, given /// that the outer abstraction pattern is a tuple. -void -ResultPlanner::planTupleFromIndirectResult(AbstractionPattern innerOrigType, - CanTupleType innerSubstType, - AbstractionPattern outerOrigType, - CanTupleType outerSubstType, - PlanData &planData, - SILValue innerResultAddr) { +void ResultPlanner::planExpandedFromIndirect(AbstractionPattern innerOrigType, + CanTupleType innerSubstType, + AbstractionPattern outerOrigType, + CanTupleType outerSubstType, + PlanData &planData, + SILValue innerResultAddr) { assert(!innerOrigType.isTuple()); assert(innerSubstType->getNumElements() == outerSubstType->getNumElements()); assert(outerOrigType.isTuple()); + assert(!outerOrigType.doesTupleVanish()); + + auto &ctx = SGF.getASTContext(); + OuterPackResultGenerator outerPacks(*this, planData); + ExpandedTupleInputGenerator + outerElt(ctx, outerPacks, outerOrigType, outerSubstType); + TupleElementAddressGenerator + innerElt(ctx, ManagedValue::forLValue(innerResultAddr), + innerOrigType, innerSubstType); + for (; !innerElt.isFinished(); innerElt.advance(), outerElt.advance()) { + assert(!outerElt.isFinished()); - for (auto eltIndex : indices(innerSubstType.getElementTypes())) { // Project the address of the element. SILValue innerEltResultAddr = - SGF.B.createTupleElementAddr(Loc, innerResultAddr, eltIndex); + innerElt.projectElementAddress(SGF, Loc).getLValueAddress(); - // Plan to expand from that location. - planFromIndirectResult(innerOrigType.getTupleElementType(eltIndex), - innerSubstType.getElementType(eltIndex), - outerOrigType.getTupleElementType(eltIndex), - outerSubstType.getElementType(eltIndex), - planData, innerEltResultAddr); + // If the outer element does not come from a pack, we just recurse. + if (!outerElt.isOrigPackExpansion()) { + planFromIndirect(innerElt.getOrigType(), innerElt.getSubstType(), + outerElt.getOrigType(), outerElt.getSubstType(), + planData, innerEltResultAddr); + continue; + } + + // Otherwise, we're going to have an indirect result for it. + auto outerEltResultAddr = + outerElt.projectPackComponent(SGF, Loc).getLValueAddress(); + + if (auto outerSubstExpansionType = + dyn_cast(outerElt.getSubstType())) { + planPackExpansionFromTuple(innerElt.getOrigType(), + cast(innerElt.getSubstType()), + outerElt.getOrigType(), + outerSubstExpansionType, + innerElt.getInducedPackType(), + innerEltResultAddr, + innerElt.getSubstElementIndex(), + outerElt.getFormalPackType(), + outerEltResultAddr, + outerElt.getPackComponentIndex()); + continue; + } + + planIndirectIntoIndirect(innerElt.getOrigType(), innerElt.getSubstType(), + outerElt.getOrigType(), outerElt.getSubstType(), + innerEltResultAddr, outerEltResultAddr); } + innerElt.finish(); + outerElt.finish(); } -void ResultPlanner::planTupleFromDirectResult(AbstractionPattern innerOrigType, - CanTupleType innerSubstType, - AbstractionPattern outerOrigType, - CanTupleType outerSubstType, - PlanData &planData, - SILResultInfo innerResult) { +void ResultPlanner::planExpandedFromDirect(AbstractionPattern innerOrigType, + CanTupleType innerSubstType, + AbstractionPattern outerOrigType, + CanTupleType outerSubstType, + PlanData &planData, + SILResultInfo innerResult) { assert(!innerOrigType.isTuple()); - auto outerSubstTupleType = dyn_cast(outerSubstType); + assert(outerOrigType.isTuple()); + assert(!outerOrigType.doesTupleVanish()); + assert(innerSubstType->getNumElements() == outerSubstType->getNumElements()); - assert(outerSubstTupleType && "Outer type must be a tuple"); - assert(innerSubstType->getNumElements() == - outerSubstTupleType->getNumElements()); - - // Create direct outer results for each of the elements. - for (auto eltIndex : indices(innerSubstType.getElementTypes())) { - AbstractionPattern newOuterOrigType = - outerOrigType.getTupleElementType(eltIndex); - AbstractionPattern newInnerOrigType = - innerOrigType.getTupleElementType(eltIndex); - if (newOuterOrigType.isTuple()) { - planTupleFromDirectResult( - newInnerOrigType, - cast(innerSubstType.getElementType(eltIndex)), - newOuterOrigType, - cast(outerSubstTupleType.getElementType(eltIndex)), - planData, innerResult); - continue; + SILType innerResultTy = SGF.getSILType(innerResult, CanSILFunctionType()); + + // If the substituted tuples contain pack expansions, we need to + // store the direct type to a temporary and then plan as if the + // result was indirect, because we can't do pack loops in SIL on + // scalar tuples. + assert(innerSubstType->containsPackExpansionType() == + outerSubstType->containsPackExpansionType()); + if (innerSubstType->containsPackExpansionType()) { + auto temporary = SGF.emitTemporaryAllocation(Loc, innerResultTy); + addDirectToIndirect(innerResult, temporary); + planExpandedFromIndirect(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + planData, temporary); + return; + } + + // Split the inner tuple value into its elements. + addDestructureDirectInnerTuple(innerResult); + + // Expand the outer tuple and recurse. + OuterPackResultGenerator outerPacks(*this, planData); + ExpandedTupleInputGenerator + outerElt(SGF.getASTContext(), outerPacks, outerOrigType, outerSubstType); + innerOrigType.forEachExpandedTupleElement(innerSubstType, + [&](AbstractionPattern innerOrigEltType, + CanType innerSubstEltType, + const TupleTypeElt &innerOrigTupleElt) { + SILType innerEltTy = + innerResultTy.getTupleElementType(outerElt.getSubstElementIndex()); + SILResultInfo innerEltResult(innerEltTy.getASTType(), + innerResult.getConvention()); + + // If the outer element comes from an orig pack expansion, it's + // always indirect. + if (outerElt.isOrigPackExpansion()) { + auto outerEltAddr = + outerElt.projectPackComponent(SGF, Loc).getLValueAddress(); + planSingleIntoIndirect(innerOrigEltType, innerSubstEltType, + outerElt.getOrigType(), outerElt.getSubstType(), + planData, innerEltResult, outerEltAddr); + + // Otherwise, we need to recurse. + } else { + planFromDirect(innerOrigEltType, innerSubstEltType, + outerElt.getOrigType(), outerElt.getSubstType(), + planData, innerEltResult); } + outerElt.advance(); + }); + outerElt.finish(); +} +void ResultPlanner::planFromDirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + PlanData &planData, + SILResultInfo innerResult) { + // If the outer type isn't a tuple, it's a single result. + if (!outerOrigType.isTuple()) { auto outerResult = claimNextOuterResult(planData); - auto elemType = outerSubstTupleType.getElementType(eltIndex); - SILResultInfo eltResult(elemType, outerResult.first.getConvention()); - planScalarIntoDirectResult( - newInnerOrigType, innerSubstType.getElementType(eltIndex), - newOuterOrigType, outerSubstTupleType.getElementType(eltIndex), - planData, eltResult, outerResult.first); + planSingle(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + planData, innerResult, outerResult.first, outerResult.second); + return; + } + + // If the outer tuple doesn't vanish, then the inner substituted type + // must have parallel tuple structure. + if (!outerOrigType.doesTupleVanish()) { + planExpandedFromDirect(innerOrigType, cast(innerSubstType), + outerOrigType, cast(outerSubstType), + planData, innerResult); + return; } + + // Otherwise, expand the outer tuple and recurse for the surviving + // element. + expandVanishingTuple(outerOrigType, outerSubstType, planData, /*inner*/ false, + [&](AbstractionPattern outerOrigEltType, CanType outerSubstEltType) { + planFromDirect(innerOrigType, innerSubstType, + outerOrigEltType, outerSubstEltType, + planData, innerResult); + }, [&](AbstractionPattern outerOrigEltType, CanType outerSubstEltType, + SILValue outerResultAddr) { + planSingleIntoIndirect(innerOrigType, innerSubstType, + outerOrigEltType, outerSubstEltType, + planData, innerResult, outerResultAddr); + }); } /// Plan the emission of a call result from an inner result address, /// given that the outer abstraction pattern is not a tuple. -void -ResultPlanner::planScalarFromIndirectResult(AbstractionPattern innerOrigType, - CanType innerSubstType, - AbstractionPattern outerOrigType, - CanType outerSubstType, - SILValue innerResultAddr, - SILResultInfo outerResult, - SILValue optOuterResultAddr) { +void ResultPlanner::planSingleFromIndirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + SILValue innerResultAddr, + SILResultInfo outerResult, + SILValue optOuterResultAddr) { assert(!innerOrigType.isTuple()); assert(!outerOrigType.isTuple()); assert(SGF.silConv.isSILIndirect(outerResult) == bool(optOuterResultAddr)); - bool hasAbstractionDifference = - (innerResultAddr->getType().getASTType() != outerResult.getInterfaceType()); - // The outer result can be indirect, and it doesn't necessarily have an // abstraction difference. Note that we should only end up in this path // in cases where simply forwarding the outer result address wasn't possible. if (SGF.silConv.isSILIndirect(outerResult)) { assert(optOuterResultAddr); - if (!hasAbstractionDifference) { - addIndirectToIndirect(innerResultAddr, optOuterResultAddr); - } else { - addReabstractIndirectToIndirect(innerOrigType, innerSubstType, - outerOrigType, outerSubstType, - innerResultAddr, optOuterResultAddr); - } + planIndirectIntoIndirect(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + innerResultAddr, optOuterResultAddr); } else { - if (!hasAbstractionDifference) { + if (!hasAbstractionDifference(innerResultAddr, outerResult)) { addIndirectToDirect(innerResultAddr, outerResult); } else { addReabstractIndirectToDirect(innerOrigType, innerSubstType, @@ -3220,45 +4148,61 @@ ResultPlanner::planScalarFromIndirectResult(AbstractionPattern innerOrigType, } } -void ResultPlanner::executeInnerTuple( - SILValue innerElement, SmallVector &innerDirectResults) { - // NOTE: We know that our value is at +1 here. - assert(innerElement->getType().getAs() && - "Only supports tuple inner types"); - - SGF.B.emitDestructureValueOperation( - Loc, innerElement, [&](unsigned index, SILValue elt) { - if (elt->getType().is()) - return executeInnerTuple(elt, innerDirectResults); - innerDirectResults.push_back(elt); - }); +void ResultPlanner::planIndirectIntoIndirect(AbstractionPattern innerOrigType, + CanType innerSubstType, + AbstractionPattern outerOrigType, + CanType outerSubstType, + SILValue innerResultAddr, + SILValue outerResultAddr) { + if (!hasAbstractionDifference(innerResultAddr, outerResultAddr)) { + addIndirectToIndirect(innerResultAddr, outerResultAddr); + } else { + addReabstractIndirectToIndirect(innerOrigType, innerSubstType, + outerOrigType, outerSubstType, + innerResultAddr, outerResultAddr); + } +} + +/// Destructure a tuple and push its elements in reverse onto +/// the given stack, so that popping them off will visit them in +/// forward order. +static void destructureAndReverseTuple(SILGenFunction &SGF, + SILLocation loc, + SILValue tupleValue, + SmallVectorImpl &values) { + auto tupleTy = tupleValue->getType().castTo(); + assert(!tupleTy->containsPackExpansionType() && + "cannot destructure a tuple with pack expansions in it"); + + SGF.B.emitDestructureValueOperation(loc, tupleValue, values); + std::reverse(values.end() - tupleTy->getNumElements(), values.end()); } -SILValue ResultPlanner::execute(SILValue innerResult) { +SILValue ResultPlanner::execute(SILValue innerResult, + CanSILFunctionType innerFnType) { // The code emission here assumes that we don't need to have // active cleanups for all the result values we're not actively // transforming. In other words, it's not "exception-safe". - // Explode the inner direct results. - SmallVector innerDirectResults; - auto innerResultTupleType = innerResult->getType().getAs(); - if (!innerResultTupleType) { - innerDirectResults.push_back(innerResult); + // Explode the first level of tuple for the direct inner results + // (the one that's introduced implicitly when there are multiple + // results). + SmallVector innerDirectResultStack; + unsigned numInnerDirectResults = + SILFunctionConventions(innerFnType, SGF.SGM.M) + .getNumDirectSILResults(); + if (numInnerDirectResults == 0) { + // silently ignore the result + } else if (numInnerDirectResults > 1) { + destructureAndReverseTuple(SGF, Loc, innerResult, + innerDirectResultStack); } else { - { - Scope S(SGF.Cleanups, CleanupLocation(Loc)); - - // First create an rvalue cleanup for our direct result. - assert(innerResult->getOwnershipKind().isCompatibleWith( - OwnershipKind::Owned)); - executeInnerTuple(innerResult, innerDirectResults); - // Then allow the cleanups to be emitted in the proper reverse order. - } + innerDirectResultStack.push_back(innerResult); } // Translate the result values. SmallVector outerDirectResults; - execute(innerDirectResults, outerDirectResults); + execute(innerDirectResultStack, outerDirectResults); // Implode the outer direct results. SILValue outerResult; @@ -3271,11 +4215,13 @@ SILValue ResultPlanner::execute(SILValue innerResult) { return outerResult; } -void ResultPlanner::execute(ArrayRef innerDirectResults, +/// innerDirectResults is a stack: we expect to pull the next result +/// off the end. +void ResultPlanner::execute(SmallVectorImpl &innerDirectResultStack, SmallVectorImpl &outerDirectResults) { // A helper function to claim an inner direct result. auto claimNextInnerDirectResult = [&](SILResultInfo result) -> ManagedValue { - auto resultValue = claimNext(innerDirectResults); + auto resultValue = innerDirectResultStack.pop_back_val(); assert(resultValue->getType() == SGF.getSILType(result, CanSILFunctionType())); auto &resultTL = SGF.getTypeLowering(result.getInterfaceType()); switch (result.getConvention()) { @@ -3400,6 +4346,10 @@ void ResultPlanner::execute(ArrayRef innerDirectResults, emitReabstract(op, /*indirect source*/ false, /*indirect dest*/ false); continue; + case Operation::ReabstractTupleIntoPackExpansion: + op.emitReabstractTupleIntoPackExpansion(SGF, Loc); + continue; + case Operation::TupleDirect: { auto firstEltIndex = outerDirectResults.size() - op.NumElements; auto elts = makeArrayRef(outerDirectResults).slice(firstEltIndex); @@ -3411,6 +4361,14 @@ void ResultPlanner::execute(ArrayRef innerDirectResults, continue; } + case Operation::DestructureDirectInnerTuple: { + auto result = claimNextInnerDirectResult(op.InnerResult); + assert(result.isPlusOne(SGF)); + destructureAndReverseTuple(SGF, Loc, result.forward(SGF), + innerDirectResultStack); + continue; + } + case Operation::InjectOptionalDirect: { SILValue value = outerDirectResults.pop_back_val(); auto tupleType = SGF.F.mapTypeIntoContext( @@ -3427,7 +4385,7 @@ void ResultPlanner::execute(ArrayRef innerDirectResults, llvm_unreachable("bad operation kind"); } - assert(innerDirectResults.empty() && "didn't consume all inner results?"); + assert(innerDirectResultStack.empty() && "didn't consume all inner results?"); } /// Build the body of a transformation thunk. @@ -3513,7 +4471,7 @@ static void buildThunkBody(SILGenFunction &SGF, SILLocation loc, /*substitutions*/ {}, argValues); // Reabstract the result. - SILValue outerResult = resultPlanner.execute(innerResult); + SILValue outerResult = resultPlanner.execute(innerResult, fnType); scope.pop(); SGF.B.createReturn(loc, outerResult); @@ -4740,7 +5698,7 @@ SILGenFunction::emitVTableThunk(SILDeclRef base, subs, args); // Reabstract the return. - result = resultPlanner->execute(implResult); + result = resultPlanner->execute(implResult, derivedFTy); break; } @@ -5114,7 +6072,7 @@ void SILGenFunction::emitProtocolWitness( emitApplyWithRethrow(loc, witnessFnRef, witnessSILTy, witnessSubs, args); // Reabstract the result value. - reqtResultValue = resultPlanner->execute(witnessResultValue); + reqtResultValue = resultPlanner->execute(witnessResultValue, witnessFTy); break; } diff --git a/lib/SILGen/TupleGenerators.h b/lib/SILGen/TupleGenerators.h new file mode 100644 index 0000000000000..120046befbeff --- /dev/null +++ b/lib/SILGen/TupleGenerators.h @@ -0,0 +1,389 @@ +//===--- TupleGenerators.h - Generators for tuple components ----*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This file defines several generator classes useful for destructuring +// tuples in various situations that arise in SIL generation. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_SILGEN_TUPLEINPUTGENERATOR_H +#define SWIFT_SILGEN_TUPLEINPUTGENERATOR_H + +#include "swift/SIL/SILValue.h" +#include "swift/SIL/AbstractionPatternGenerators.h" +#include "swift/AST/Types.h" +#include "swift/Basic/ExternalUnion.h" +#include "swift/Basic/Generators.h" + +namespace swift { +namespace Lowering { + +/// A generator for destructuring a tuple type that is recursively +/// expanded in some sequence. In SIL, this sort of expansion +/// happens in both function parameters and function results. +/// The expectation is that the client will use this generator +/// to walk the sequence and then handle the element cases itself. +/// +/// Unlike TupleElementGenerator, the iteration visits each +/// individual element of the substituted tuple type. This makes +/// this generator more appropriate for use cases where the elements +/// must all be visited, such as when iterating the elements of two +/// tuples in paralle. +/// +/// In order to properly handle pack expansions within tuples, +/// especially any empty pack expansions in the original tuple type, +/// this generator must claim the packs as it goes. Since pack +/// values are typically claimed from a sequence that interleaves +/// packs with other values, it is important that clients claim +/// non-pack values from the sequence immediately when iteration +/// reaches them. +class ExpandedTupleInputGenerator { + const ASTContext &ctx; + SimpleGeneratorRef packInputs; + TupleElementGenerator origElt; + + /// If origElt.isPackExpansion(), this is the pack currently + /// being destructured. + ManagedValue packValue; + + /// If origElt.isPackExpansion(), this is the formal type + /// of the orig pack. + CanPackType formalPackType; + + /// The current index within origElt.getSubstEltTypes(). + unsigned substEltIndex; + + /// Precondition: we aren't finished visiting orig elements, + /// and we've already readied the current orig element. + /// + /// Ready the next subst element (the one at substEltIndex) + /// from the current orig element. If we've exhausted the supply + /// of subst elements from this orig element, advance to the + /// next orig element and repeat. + /// + /// Postcondition: we're either finished or properly configured + /// with a subst element that hasn't been presented before. + void readyNextSubstElement() { + while (true) { + assert(!origElt.isFinished()); + assert(substEltIndex <= origElt.getSubstTypes().size()); + + // If we haven't reached the limit of the current element yet, + // continue. + if (substEltIndex != origElt.getSubstTypes().size()) + return; + + // Otherwise, advance, and ready the next orig element if we + // didn't finish. + origElt.advance(); + if (origElt.isFinished()) return; + readyOrigElement(); + } + } + + /// Ready the current orig element. + void readyOrigElement() { + substEltIndex = 0; + if (origElt.isOrigPackExpansion()) { + // The pack value exists in the lowered inputs and must be + // claimed whether it contains formal elements or not. + packValue = packInputs.claimNext(); + + // We don't need to do any other set up if we're going to + // immediately move past it, though. + if (origElt.getSubstTypes().empty()) + return; + + // Compute a formal pack type for the pack. + formalPackType = CanPackType::get(ctx, origElt.getSubstTypes()); + } + } + + void updatePackValue(ManagedValue newPackValue) { + assert(isOrigPackExpansion()); + packValue = newPackValue; + } + +public: + ExpandedTupleInputGenerator(const ASTContext &ctx, + SimpleGeneratorRef inputs, + AbstractionPattern origTupleType, + CanType substType) + : ctx(ctx), packInputs(inputs), origElt(origTupleType, substType) { + + if (!origElt.isFinished()) { + readyOrigElement(); + readyNextSubstElement(); + } + } + + /// Is this generator finished? If so, the getters below may not be used. + bool isFinished() const { + return origElt.isFinished(); + } + + bool doesOrigTupleVanish() const { + return origElt.doesOrigTupleVanish(); + } + + bool isOrigPackExpansion() const { + return origElt.isOrigPackExpansion(); + } + + AbstractionPattern getOrigType() const { + return origElt.isOrigPackExpansion() + ? origElt.getOrigType().getPackExpansionPatternType() + : origElt.getOrigType(); + } + + /// Return the index of the current element in the substituted + /// tuple type. + unsigned getSubstElementIndex() const { + return origElt.getSubstIndex() + substEltIndex; + } + + CanType getSubstType() const { + return origElt.getSubstTypes()[substEltIndex]; + } + + bool isSubstPackExpansion() const { + return isa(getSubstType()); + } + + ManagedValue getPackValue() const { + assert(isOrigPackExpansion()); + return packValue; + } + + unsigned getPackComponentIndex() const { + assert(isOrigPackExpansion()); + return substEltIndex; + } + + CanPackType getFormalPackType() const { + assert(isOrigPackExpansion()); + return formalPackType; + } + + /// Given that we just processed an input, advance to the next, + /// if there is on. + /// + /// Postcondition: either isFinished() or all of the invariants + /// for the mutable state have been established. + void advance() { + assert(!isFinished()); + assert(substEltIndex < origElt.getSubstTypes().size()); + substEltIndex++; + + readyNextSubstElement(); + } + + void finish() { + origElt.finish(); + } + + /// Project out the current pack component. Do not call this multiple + /// times for the same component. + /// + /// You should only call this when the pack is already initialized, + /// which generally means when it corresponds to an input you've + /// received. In the reabstraction code, this means it should only + /// be used for *outer* types. If the pack is part of an output + /// you're generating, you should call createPackComponentTemporary. + ManagedValue projectPackComponent(SILGenFunction &SGF, SILLocation loc); + + /// Create a temporary for the current pack component and set it in the + /// pack. Do not call this multiple times for the same component. + /// + /// You should only call this when the pack not yet initialized, + /// which generally means when it corresponds to an output you're + /// generating. In the reabstraction code, this means it should only + /// be used for *inner* types. If the pack is an input you've + /// received, you should call projectPackComponent. + SILValue createPackComponentTemporary(SILGenFunction &SGF, SILLocation loc); +}; + +/// A generator for visiting the addresses of the elements of +/// a tuple. Unlike the other tuple generators, this does not +/// require the original abstraction pattern to be a tuple pattern: +/// like forEachExpandedTupleElement, it permits an opaque +/// abstraction pattern. +class TupleElementAddressGenerator { + struct OpaquePatternStorage { + AbstractionPattern origType; + CanTupleType substType; + }; + using Members = ExternalUnionMembers; + static Members::Index getIndexForKind(bool isOpaque) { + return isOpaque ? Members::indexOf() + : Members::indexOf(); + } + ExternalUnion origElt; + + /// The address of the tuple value. + ManagedValue tupleAddr; + + /// If the substituted tuple type contains pack expansions, this is + /// the induced pack type for the element sequence. + CanPackType inducedPackType; + + /// The current index within origElt.getSubstEltTypes(). + unsigned substEltIndex; + + /// Whether the orig type is opaque, and therefore whether origElt + /// is storing an OpaquePatternStorage or a TupleElementGenerator. + bool isOrigTypeOpaque; + + /// Precondition: we aren't finished visiting orig elements, + /// and we've already readied the current orig element. + /// + /// Ready the next subst element (the one at substEltIndex) + /// from the current orig element. If we've exhausted the supply + /// of subst elements from this orig element, advance to the + /// next orig element and repeat. + /// + /// Postcondition: we're either finished or properly configured + /// with a subst element that hasn't been presented before. + void readyNextSubstElement() { + auto &gen = origElt.get(isOrigTypeOpaque); + while (true) { + assert(!gen.isFinished()); + assert(substEltIndex <= gen.getSubstTypes().size()); + + // If we haven't reached the limit of the current element yet, + // continue. + if (substEltIndex != gen.getSubstTypes().size()) + return; + + // Otherwise, advance, and ready the next orig element if we + // didn't finish. + gen.advance(); + if (gen.isFinished()) return; + substEltIndex = 0; + } + } + +public: + TupleElementAddressGenerator(const ASTContext &ctx, + ManagedValue tupleAddr, + AbstractionPattern origType, + CanTupleType substType) + : tupleAddr(tupleAddr) { + + if (substType->containsPackExpansionType()) { + inducedPackType = CanPackType::get(ctx, substType.getElementTypes()); + } + + isOrigTypeOpaque = !origType.isTuple(); + substEltIndex = 0; + if (isOrigTypeOpaque) { + assert(origType.isTypeParameterOrOpaqueArchetype()); + origElt.emplaceAggregate(isOrigTypeOpaque, + origType, substType); + } else { + auto &gen = + origElt.emplace(isOrigTypeOpaque, + origType, substType); + if (!gen.isFinished()) { + readyNextSubstElement(); + } + } + } + + /// Is this generator finished? If so, the getters below may not be used. + bool isFinished() const { + if (isOrigTypeOpaque) { + auto &storage = origElt.get(isOrigTypeOpaque); + return substEltIndex == storage.substType->getNumElements(); + } else { + return origElt.get(isOrigTypeOpaque).isFinished(); + } + } + + AbstractionPattern getOrigType() const { + if (isOrigTypeOpaque) { + return origElt.get(isOrigTypeOpaque).origType; + } else { + auto &gen = origElt.get(isOrigTypeOpaque); + return gen.isOrigPackExpansion() + ? gen.getOrigType().getPackExpansionPatternType() + : gen.getOrigType(); + } + } + + CanType getSubstType() const { + if (isOrigTypeOpaque) { + auto &opaque = origElt.get(isOrigTypeOpaque); + return opaque.substType.getElementType(substEltIndex); + } else { + auto &gen = origElt.get(isOrigTypeOpaque); + return gen.getSubstTypes()[substEltIndex]; + } + } + + bool isSubstPackExpansion() const { + return isa(getSubstType()); + } + + unsigned getSubstElementIndex() const { + if (isOrigTypeOpaque) { + return substEltIndex; + } else { + auto &gen = origElt.get(isOrigTypeOpaque); + return gen.getSubstIndex() + substEltIndex; + } + } + + bool tupleContainsPackExpansion() const { + return (bool) inducedPackType; + } + + CanPackType getInducedPackType() const { + assert(tupleContainsPackExpansion()); + return inducedPackType; + } + + /// Given that we just processed an input, advance to the next, + /// if there is on. + /// + /// Postcondition: either isFinished() or all of the invariants + /// for the mutable state have been established. + void advance() { + assert(!isFinished()); + substEltIndex++; + if (!isOrigTypeOpaque) { + readyNextSubstElement(); + } + } + + void finish() { + if (isOrigTypeOpaque) { +#ifndef NDEBUG + auto &opaque = origElt.get(isOrigTypeOpaque); + assert(substEltIndex == opaque.substType->getNumElements()); +#endif + } else { + auto &gen = origElt.get(isOrigTypeOpaque); + gen.finish(); + } + } + + /// Project out the current tuple element. Call this exactly once + /// per element. + ManagedValue projectElementAddress(SILGenFunction &SGF, SILLocation loc); +}; + +} // end namespace Lowering +} // end namespace swift + +#endif diff --git a/test/SILGen/variadic-generic-reabstract-tuple-result.swift b/test/SILGen/variadic-generic-reabstract-tuple-result.swift new file mode 100644 index 0000000000000..bc82280f61fd7 --- /dev/null +++ b/test/SILGen/variadic-generic-reabstract-tuple-result.swift @@ -0,0 +1,327 @@ +// RUN: %target-swift-emit-silgen -disable-availability-checking %s | %FileCheck %s + +// rdar://110391963 + +struct Use {} + +struct FunctionProducer { + static func get() -> () -> (repeat each T) { + return { preconditionFailure("") } + } +} + +struct PlusFunctionProducer { + static func get() -> () -> (U, repeat each T) { + return { preconditionFailure("") } + } +} + +struct FunctionConsumer { + static func set(_: () -> (repeat each T)) {} +} + +struct PlusFunctionConsumer { + static func set(_: () -> (U, repeat each T)) {} +} + +func test1() { + let fn = FunctionProducer.get() + _ = fn() +} +// CHECK-LABEL: sil {{.*}} @$s4main5test1yyF : +// CHECK: function_ref @$sSS_QSiIegk_SSIego_TR : $@convention(thin) (@guaranteed @callee_guaranteed () -> @pack_out Pack{String}) -> @owned String +// CHECK-LABEL: sil shared {{.*}} @$sSS_QSiIegk_SSIego_TR : +// CHECK: [[PACK:%.*]] = alloc_pack $Pack{String} +// CHECK-NEXT: [[STRING_TEMP:%.*]] = alloc_stack $String +// CHECK-NEXT: [[STRING_INDEX:%.*]] = scalar_pack_index 0 of $Pack{String} +// CHECK-NEXT: pack_element_set [[STRING_TEMP]] : $*String into [[STRING_INDEX]] of [[PACK]] : $*Pack{String} +// CHECK-NEXT: apply %0([[PACK]]) +// CHECK-NEXT: [[STRING:%.*]] = load [take] [[STRING_TEMP]] : $*String +// CHECK-NEXT: dealloc_stack [[STRING_TEMP]] : +// CHECK-NEXT: dealloc_pack [[PACK]] : +// CHECK-NEXT: return [[STRING]] + +func test2() { + let fn = FunctionProducer.get() + _ = fn() +} +// CHECK-LABEL: sil {{.*}} @$s4main5test2yyF : +// CHECK: function_ref @$sSS_SiQSiIegk_SSSiIegod_TR : $@convention(thin) (@guaranteed @callee_guaranteed () -> @pack_out Pack{String, Int}) -> (@owned String, Int) +// CHECK: sil shared {{.*}} @$sSS_SiQSiIegk_SSSiIegod_TR : +// CHECK: [[PACK:%.*]] = alloc_pack $Pack{String, Int} +// CHECK-NEXT: [[STRING_TEMP:%.*]] = alloc_stack $String +// CHECK-NEXT: [[STRING_INDEX:%.*]] = scalar_pack_index 0 of $Pack{String, Int} +// CHECK-NEXT: pack_element_set [[STRING_TEMP]] : $*String into [[STRING_INDEX]] of [[PACK]] : $*Pack{String, Int} +// CHECK-NEXT: [[INT_TEMP:%.*]] = alloc_stack $Int +// CHECK-NEXT: [[INT_INDEX:%.*]] = scalar_pack_index 1 of $Pack{String, Int} +// CHECK-NEXT: pack_element_set [[INT_TEMP]] : $*Int into [[INT_INDEX]] of [[PACK]] : $*Pack{String, Int} +// CHECK-NEXT: apply %0([[PACK]]) +// CHECK-NEXT: [[STRING:%.*]] = load [take] [[STRING_TEMP]] : $*String +// CHECK-NEXT: [[INT:%.*]] = load [trivial] [[INT_TEMP]] : $*Int +// CHECK-NEXT: [[TUPLE:%.*]] = tuple ([[STRING]] : $String, [[INT]] : $Int) +// CHECK-NEXT: dealloc_stack [[INT_TEMP]] : +// CHECK-NEXT: dealloc_stack [[STRING_TEMP]] : +// CHECK-NEXT: dealloc_pack [[PACK]] : +// CHECK-NEXT: return [[TUPLE]] + +func test3() { + let fn = FunctionProducer< >.get() + _ = fn() +} +// CHECK-LABEL: sil {{.*}} @$s4main5test3yyF : +// CHECK: function_ref @$syQSiIegk_Ieg_TR : $@convention(thin) (@guaranteed @callee_guaranteed () -> @pack_out Pack{}) -> () +// CHECK: sil shared {{.*}} @$syQSiIegk_Ieg_TR : +// CHECK: [[PACK:%.*]] = alloc_pack $Pack{} +// CHECK-NEXT: apply %0([[PACK]]) +// CHECK-NEXT: [[TUPLE:%.*]] = tuple () +// CHECK-NEXT: dealloc_pack [[PACK]] : +// CHECK-NEXT: return [[TUPLE]] + +func test4() { + let fn = PlusFunctionProducer.get() + _ = fn() +} +// CHECK-LABEL: sil {{.*}} @$s4main5test4yyF : +// CHECK: function_ref @$sSSyQSiIegrk_SSIego_TR : $@convention(thin) (@guaranteed @callee_guaranteed () -> (@out String, @pack_out Pack{})) -> @owned String +// CHECK-LABEL: sil shared {{.*}} @$sSSyQSiIegrk_SSIego_TR : +// CHECK: [[STRING_TEMP:%.*]] = alloc_stack $String +// CHECK-NEXT: [[PACK:%.*]] = alloc_pack $Pack{} +// CHECK-NEXT: apply %0([[STRING_TEMP]], [[PACK]]) +// CHECK-NEXT: [[STRING:%.*]] = load [take] [[STRING_TEMP]] : $*String +// CHECK-NEXT: dealloc_pack [[PACK]] : +// CHECK-NEXT: dealloc_stack [[STRING_TEMP]] : +// CHECK-NEXT: return [[STRING]] + +func test5(fn: () -> (String, Int)) { + FunctionConsumer.set(fn) +} +// CHECK-LABEL: sil {{.*}} @$s4main5test52fnySS_SityXE_tF : +// CHECK: function_ref @$sSSSiIgod_SS_SiQSiIegk_TR : $@convention(thin) (@guaranteed @noescape @callee_guaranteed () -> (@owned String, Int)) -> @pack_out Pack{String, Int} +// CHECK-LABEL: sil shared {{.*}} @$sSSSiIgod_SS_SiQSiIegk_TR : +// CHECK: [[STRING_INDEX:%.*]] = scalar_pack_index 0 of $Pack{String, Int} +// CHECK-NEXT: [[STRING_ADDR:%.*]] = pack_element_get [[STRING_INDEX]] of %0 : $*Pack{String, Int} as $*String +// CHECK-NEXT: [[INT_INDEX:%.*]] = scalar_pack_index 1 of $Pack{String, Int} +// CHECK-NEXT: [[INT_ADDR:%.*]] = pack_element_get [[INT_INDEX]] of %0 : $*Pack{String, Int} as $*Int +// CHECK-NEXT: [[RESULT:%.*]] = apply %1() +// CHECK-NEXT: ([[STRING:%.*]], [[INT:%.*]]) = destructure_tuple [[RESULT]] +// CHECK-NEXT: store [[STRING]] to [init] [[STRING_ADDR]] : +// CHECK-NEXT: store [[INT]] to [trivial] [[INT_ADDR]] : + +func test6(fn: () -> (String, Int)) { + PlusFunctionConsumer.set(fn) +} +// CHECK-LABEL: sil {{.*}} @$s4main5test62fnySS_SityXE_tF : +// CHECK: function_ref @$sSSSiIgod_SSSi_QSiIegrk_TR : $@convention(thin) (@guaranteed @noescape @callee_guaranteed () -> (@owned String, Int)) -> (@out String, @pack_out Pack{Int}) +// CHECK-LABEL: sil shared {{.*}} @$sSSSiIgod_SSSi_QSiIegrk_TR : +// CHECK: [[INT_INDEX:%.*]] = scalar_pack_index 0 of $Pack{Int} +// CHECK-NEXT: [[INT_ADDR:%.*]] = pack_element_get [[INT_INDEX]] of %1 : $*Pack{Int} as $*Int +// CHECK-NEXT: [[RESULT:%.*]] = apply %2() +// CHECK-NEXT: ([[STRING:%.*]], [[INT:%.*]]) = destructure_tuple [[RESULT]] +// CHECK-NEXT: store [[STRING]] to [init] %0 : +// CHECK-NEXT: store [[INT]] to [trivial] [[INT_ADDR]] : + +func test7(fn: () -> String) { + PlusFunctionConsumer.set(fn) +} +// CHECK-LABEL: sil {{.*}} @$s4main5test72fnySSyXE_tF : +// CHECK: function_ref @$sSSIgo_SSyQSiIegrk_TR : $@convention(thin) (@guaranteed @noescape @callee_guaranteed () -> @owned String) -> (@out String, @pack_out Pack{}) +// CHECK-LABEL: sil shared {{.*}} @$sSSIgo_SSyQSiIegrk_TR : +// CHECK: [[STRING:%.*]] = apply %2() +// CHECK-NEXT: store [[STRING]] to [init] %0 : + +func test8() { + PlusFunctionConsumer.set(FunctionProducer<(String, Int)>.get()) +} +// CHECK-LABEL: sil {{.*}} @$s4main5test8yyF : +// We emit this with a chain of two reabstraction thunks, which is a pity, +// because otherwise it would truly be a marvelously complex thunk. +// CHECK: function_ref @$sSS_Sit_QSiIegk_SSSiIegod_TR : $@convention(thin) (@guaranteed @callee_guaranteed () -> @pack_out Pack{(String, Int)}) -> (@owned String, Int) +// CHECK: function_ref @$sSSSiIgod_SSSi_QSiIegrk_TR : $@convention(thin) (@guaranteed @noescape @callee_guaranteed () -> (@owned String, Int)) -> (@out String, @pack_out Pack{Int}) +// CHECK-LABEL: sil shared {{.*}} @$sSS_Sit_QSiIegk_SSSiIegod_TR : +// CHECK: [[PACK:%.*]] = alloc_pack $Pack{(String, Int)} +// CHECK-NEXT: [[TUPLE_TEMP:%.*]] = alloc_stack $(String, Int) +// CHECK-NEXT: [[TUPLE_INDEX:%.*]] = scalar_pack_index 0 of $Pack{(String, Int)} +// CHECK-NEXT: pack_element_set [[TUPLE_TEMP]] : $*(String, Int) into [[TUPLE_INDEX]] of [[PACK]] : $*Pack{(String, Int)} +// CHECK-NEXT: [[STRING_ADDR:%.*]] = tuple_element_addr [[TUPLE_TEMP]] : $*(String, Int), 0 +// CHECK-NEXT: [[INT_ADDR:%.*]] = tuple_element_addr [[TUPLE_TEMP]] : $*(String, Int), 1 +// CHECK-NEXT: apply %0([[PACK]]) +// CHECK-NEXT: [[STRING:%.*]] = load [take] [[STRING_ADDR]] : $*String +// CHECK-NEXT: [[INT:%.*]] = load [trivial] [[INT_ADDR]] : $*Int +// CHECK-NEXT: [[TUPLE:%.*]] = tuple ([[STRING]] : $String, [[INT]] : $Int) +// CHECK-NEXT: dealloc_stack [[TUPLE_TEMP]] : +// CHECK-NEXT: dealloc_pack [[PACK]] : +// CHECK-NEXT: return [[TUPLE]] + +func test9() -> Use { + let fn = FunctionProducer.get() + _ = fn() + return Use() +} +// CHECK-LABEL: sil {{.*}} @$s4main5test9AA3UseVyxxQp_QPGyRvzlF : +// CHECK: function_ref @$sSS_xxQpSiQSiIegk_SSxxQp_QSiSiIegokd_RvzlTR : $@convention(thin) (@guaranteed @callee_guaranteed () -> @pack_out Pack{String, repeat each τ_0_0, Int}) -> (@owned String, @pack_out Pack{repeat each τ_0_0}, Int) +// CHECK-LABEL: sil shared {{.*}} @$sSS_xxQpSiQSiIegk_SSxxQp_QSiSiIegokd_RvzlTR : +// Create the inner pack argument. +// CHECK: [[PACK:%.*]] = alloc_pack $Pack{String, repeat each T, Int} +// - Set up the string result temporary. +// CHECK-NEXT: [[STRING_TEMP:%.*]] = alloc_stack $String +// CHECK-NEXT: [[STRING_INDEX:%.*]] = scalar_pack_index 0 of $Pack{String, repeat each T, Int} +// CHECK-NEXT: pack_element_set [[STRING_TEMP]] : $*String into [[STRING_INDEX]] of [[PACK]] : $*Pack{String, repeat each T, Int} +// - Forward the pack expansion addresses. +// CHECK-NEXT: [[ZERO:%.*]] = integer_literal $Builtin.Word, 0 +// CHECK-NEXT: [[ONE:%.*]] = integer_literal $Builtin.Word, 1 +// CHECK-NEXT: [[LEN:%.*]] = pack_length $Pack{repeat each T} +// CHECK-NEXT: br bb1([[ZERO]] : $Builtin.Word) +// CHECK: bb1([[IDX:%.*]] : $Builtin.Word) +// CHECK-NEXT: [[IDX_EQ_LEN:%.*]] = builtin "cmp_eq_Word"([[IDX]] : $Builtin.Word, [[LEN]] : $Builtin.Word) : $Builtin.Int1 +// CHECK-NEXT: cond_br [[IDX_EQ_LEN]], bb3, bb2 +// CHECK: bb2: +// CHECK-NEXT: [[EXPANSION_INDEX:%.*]] = dynamic_pack_index [[IDX]] of $Pack{repeat each T} +// CHECK-NEXT: open_pack_element [[EXPANSION_INDEX]] of at , shape $each T, uuid [[UUID:".*"]] +// CHECK-NEXT: [[ELT_INDEX:%.*]] = pack_pack_index 1, [[EXPANSION_INDEX]] of $Pack{String, repeat each T, Int} +// CHECK-NEXT: [[DEST_ELT_ADDR:%.*]] = pack_element_get [[EXPANSION_INDEX]] of %0 : $*Pack{repeat each T} as $*@pack_element([[UUID]]) each T +// CHECK-NEXT: pack_element_set [[DEST_ELT_ADDR]] : $*@pack_element([[UUID]]) each T into [[ELT_INDEX]] of [[PACK]] : $*Pack{String, repeat each T, Int} +// CHECK-NEXT: [[NEXT_IDX:%.*]] = builtin "add_Word"([[IDX]] : $Builtin.Word, [[ONE]] : $Builtin.Word) : $Builtin.Word +// CHECK-NEXT: br bb1([[NEXT_IDX]] : $Builtin.Word) +// CHECK: bb3: +// - Set up the int result temporary. +// CHECK-NEXT: [[INT_TEMP:%.*]] = alloc_stack $Int +// CHECK-NEXT: [[INT_INDEX:%.*]] = scalar_pack_index 2 of $Pack{String, repeat each T, Int} +// CHECK-NEXT: pack_element_set [[INT_TEMP]] : $*Int into [[INT_INDEX]] of [[PACK]] : $*Pack{String, repeat each T, Int} +// Call the reabstracted function. +// CHECK-NEXT: apply %1([[PACK]]) +// Load the direct components and tuple them up for the return. +// CHECK-NEXT: [[STRING:%.*]] = load [take] [[STRING_TEMP]] : $*String +// CHECK-NEXT: [[INT:%.*]] = load [trivial] [[INT_TEMP]] : $*Int +// CHECK-NEXT: [[TUPLE:%.*]] = tuple ([[STRING]] : $String, [[INT]] : $Int) +// CHECK-NEXT: dealloc_stack [[INT_TEMP]] : +// CHECK-NEXT: dealloc_stack [[STRING_TEMP]] : +// CHECK-NEXT: dealloc_pack [[PACK]] : +// CHECK-NEXT: return [[TUPLE]] + +func test10() -> Use { + let fn = FunctionProducer<(String, repeat each T, Int)>.get() + _ = fn() + return Use() +} +// CHECK-LABEL: sil {{.*}} @$s4main6test10AA3UseVyxxQp_QPGyRvzlF : +// CHECK: function_ref @$sSS_xxQpSit_QSiIegk_SSxxQp_QSiSiIegokd_RvzlTR : $@convention(thin) (@guaranteed @callee_guaranteed () -> @pack_out Pack{(String, repeat each τ_0_0, Int)}) -> (@owned String, @pack_out Pack{repeat each τ_0_0}, Int) +// CHECK-LABEL: sil shared {{.*}} @$sSS_xxQpSit_QSiIegk_SSxxQp_QSiSiIegokd_RvzlTR : +// Set up the inner pack argument with a tuple temporary. +// CHECK: [[PACK:%.*]] = alloc_pack $Pack{(String, repeat each T, Int)} +// CHECK-NEXT: [[TUPLE_TEMP:%.*]] = alloc_stack $(String, repeat each T, Int) +// CHECK-NEXT: [[TUPLE_INDEX:%.*]] = scalar_pack_index 0 of $Pack{(String, repeat each T, Int)} +// CHECK-NEXT: pack_element_set [[TUPLE_TEMP]] : $*(String, repeat each T, Int) into [[TUPLE_INDEX]] of [[PACK]] : $*Pack{(String, repeat each T, Int)} +// Project out the tuple elements for the non-expansion elements; +// we'll use these after the call. +// CHECK-NEXT: [[STRING_INDEX:%.*]] = scalar_pack_index 0 of $Pack{String, repeat each T, Int} +// CHECK-NEXT: [[STRING_ADDR:%.*]] = tuple_pack_element_addr [[STRING_INDEX]] of [[TUPLE_TEMP]] : $*(String, repeat each T, Int) as $*String +// CHECK-NEXT: [[INT_INDEX:%.*]] = scalar_pack_index 2 of $Pack{String, repeat each T, Int} +// CHECK-NEXT: [[INT_ADDR:%.*]] = tuple_pack_element_addr [[INT_INDEX]] of [[TUPLE_TEMP]] : $*(String, repeat each T, Int) as $*Int +// Call the reabstracted function. +// CHECK-NEXT: apply %1([[PACK]]) +// Load the first component of the tuple (the string). +// CHECK-NEXT: [[STRING:%.*]] = load [take] [[STRING_ADDR]] : $*String +// Perform a pack loop to move the pack expansion elements into +// the outer pack. +// CHECK-NEXT: [[ZERO:%.*]] = integer_literal $Builtin.Word, 0 +// CHECK-NEXT: [[ONE:%.*]] = integer_literal $Builtin.Word, 1 +// CHECK-NEXT: [[LEN:%.*]] = pack_length $Pack{repeat each T} +// CHECK-NEXT: br bb1([[ZERO]] : $Builtin.Word) +// CHECK: bb1([[IDX:%.*]] : $Builtin.Word) +// CHECK-NEXT: [[IDX_EQ_LEN:%.*]] = builtin "cmp_eq_Word"([[IDX]] : $Builtin.Word, [[LEN]] : $Builtin.Word) : $Builtin.Int1 +// CHECK-NEXT: cond_br [[IDX_EQ_LEN]], bb3, bb2 +// CHECK: bb2: +// CHECK-NEXT: [[EXPANSION_INDEX:%.*]] = dynamic_pack_index [[IDX]] of $Pack{repeat each T} +// CHECK-NEXT: open_pack_element [[EXPANSION_INDEX]] of at , shape $each T, uuid [[UUID:".*"]] +// CHECK-NEXT: [[TUPLE_ELT_INDEX:%.*]] = pack_pack_index 1, [[EXPANSION_INDEX]] of $Pack{String, repeat each T, Int} +// CHECK-NEXT: [[SRC_ELT_ADDR:%.*]] = tuple_pack_element_addr [[TUPLE_ELT_INDEX]] of [[TUPLE_TEMP]] : $*(String, repeat each T, Int) as $*@pack_element([[UUID]]) each T +// CHECK-NEXT: [[DEST_ELT_ADDR:%.*]] = pack_element_get [[EXPANSION_INDEX]] of %0 : $*Pack{repeat each T} as $*@pack_element([[UUID]]) each T +// CHECK-NEXT: copy_addr [take] [[SRC_ELT_ADDR]] to [init] [[DEST_ELT_ADDR]] : $*@pack_element([[UUID]]) each T +// CHECK-NEXT: [[NEXT_IDX:%.*]] = builtin "add_Word"([[IDX]] : $Builtin.Word, [[ONE]] : $Builtin.Word) : $Builtin.Word +// CHECK-NEXT: br bb1([[NEXT_IDX]] : $Builtin.Word) +// CHECK: bb3: +// Load the last component of the tuple (the int). +// CHECK-NEXT: [[INT:%.*]] = load [trivial] [[INT_ADDR]] : $*Int +// Return. +// CHECK-NEXT: [[TUPLE:%.*]] = tuple ([[STRING]] : $String, [[INT]] : $Int) +// CHECK-NEXT: dealloc_stack [[TUPLE_TEMP]] : +// CHECK-NEXT: dealloc_pack [[PACK]] : +// CHECK-NEXT: return [[TUPLE]] + +// Test that we can reabstract the components of a pack expansion. +func test11() -> Use { + let fn = FunctionProducer Bool, Int>.get() + _ = fn() + return Use() +} +// CHECK-LABEL: sil {{.*}} @$s4main6test11AA3UseVyxxQp_QPGyRvzlF : +// CHECK: function_ref @$sSS_xq_r0_lyxSbIsegnr_xQpSiQSiIegk_SSxSblyxIsegnd_xQp_QSiSiIegokd_RvzlTR : $@convention(thin) (@guaranteed @callee_guaranteed () -> @pack_out Pack{String, repeat @callee_guaranteed @substituted <τ_0_0, τ_0_1> (@in_guaranteed τ_0_0) -> @out τ_0_1 for , Int}) -> (@owned String, @pack_out Pack{repeat @callee_guaranteed @substituted <τ_0_0> (@in_guaranteed τ_0_0) -> Bool for }, Int) +// CHECK-LABEL: sil shared {{.*}} @$sSS_xq_r0_lyxSbIsegnr_xQpSiQSiIegk_SSxSblyxIsegnd_xQp_QSiSiIegokd_RvzlTR : +// Create the inner pack argument. +// CHECK: [[PACK:%.*]] = alloc_pack $Pack{String, repeat @callee_guaranteed @substituted <τ_0_0, τ_0_1> (@in_guaranteed τ_0_0) -> @out τ_0_1 for , Int} +// - Set up the string result temporary. +// CHECK-NEXT: [[STRING_TEMP:%.*]] = alloc_stack $String +// CHECK-NEXT: [[STRING_INDEX:%.*]] = scalar_pack_index 0 of $Pack{String, repeat (each T) -> Bool, Int} +// CHECK-NEXT: pack_element_set [[STRING_TEMP]] : $*String into [[STRING_INDEX]] of [[PACK]] : $*Pack{String, repeat @callee_guaranteed @substituted <τ_0_0, τ_0_1> (@in_guaranteed τ_0_0) -> @out τ_0_1 for , Int} +// - Set up a temporary for the pack expansion values and project +// its element addresses into the pack. +// CHECK-NEXT: [[EXPANSION_TEMP:%.*]] = alloc_stack $(repeat @callee_guaranteed @substituted <τ_0_0, τ_0_1> (@in_guaranteed τ_0_0) -> @out τ_0_1 for ) +// CHECK-NEXT: [[ZERO:%.*]] = integer_literal $Builtin.Word, 0 +// CHECK-NEXT: [[ONE:%.*]] = integer_literal $Builtin.Word, 1 +// CHECK-NEXT: [[LEN:%.*]] = pack_length $Pack{repeat each T} +// CHECK-NEXT: br bb1([[ZERO]] : $Builtin.Word) +// CHECK: bb1([[IDX:%.*]] : $Builtin.Word) +// CHECK-NEXT: [[IDX_EQ_LEN:%.*]] = builtin "cmp_eq_Word"([[IDX]] : $Builtin.Word, [[LEN]] : $Builtin.Word) : $Builtin.Int1 +// CHECK-NEXT: cond_br [[IDX_EQ_LEN]], bb3, bb2 +// CHECK: bb2: +// CHECK-NEXT: [[EXPANSION_INDEX:%.*]] = dynamic_pack_index [[IDX]] of $Pack{repeat (each T) -> Bool} +// CHECK-NEXT: open_pack_element [[EXPANSION_INDEX]] of at , shape $each T, uuid [[UUID:".*"]] +// CHECK-NEXT: [[PACK_ELT_INDEX:%.*]] = pack_pack_index 1, [[EXPANSION_INDEX]] of $Pack{String, repeat (each T) -> Bool, Int} +// CHECK-NEXT: [[TEMP_ELT_ADDR:%.*]] = tuple_pack_element_addr [[EXPANSION_INDEX]] of [[EXPANSION_TEMP]] : $*(repeat @callee_guaranteed @substituted <τ_0_0, τ_0_1> (@in_guaranteed τ_0_0) -> @out τ_0_1 for ) as $*@callee_guaranteed @substituted <τ_0_0, τ_0_1> (@in_guaranteed τ_0_0) -> @out τ_0_1 for <@pack_element([[UUID]]) each T, Bool> +// CHECK-NEXT: pack_element_set [[TEMP_ELT_ADDR]] : $*@callee_guaranteed @substituted <τ_0_0, τ_0_1> (@in_guaranteed τ_0_0) -> @out τ_0_1 for <@pack_element([[UUID]]) each T, Bool> into [[PACK_ELT_INDEX]] of [[PACK]] : +// CHECK-NEXT: [[NEXT_IDX:%.*]] = builtin "add_Word"([[IDX]] : $Builtin.Word, [[ONE]] : $Builtin.Word) : $Builtin.Word +// CHECK-NEXT: br bb1([[NEXT_IDX]] : $Builtin.Word) +// CHECK: bb3: +// - Set up the int result temporary. +// CHECK-NEXT: [[INT_TEMP:%.*]] = alloc_stack $Int +// CHECK-NEXT: [[INT_INDEX:%.*]] = scalar_pack_index 2 of $Pack{String, repeat (each T) -> Bool, Int} +// CHECK-NEXT: pack_element_set [[INT_TEMP]] : $*Int into [[INT_INDEX]] of [[PACK]] : $*Pack{String, repeat @callee_guaranteed @substituted <τ_0_0, τ_0_1> (@in_guaranteed τ_0_0) -> @out τ_0_1 for , Int} +// Call the reabstracted function. +// CHECK-NEXT: apply %1([[PACK]]) +// Reassemble the outer results: +// - Load the first direct component. +// CHECK-NEXT: [[STRING:%.*]] = load [take] [[STRING_TEMP]] : $*String +// - Reabstract the pack expansion components. +// CHECK-NEXT: [[ZERO:%.*]] = integer_literal $Builtin.Word, 0 +// CHECK-NEXT: [[ONE:%.*]] = integer_literal $Builtin.Word, 1 +// CHECK-NEXT: [[LEN:%.*]] = pack_length $Pack{repeat each T} +// CHECK-NEXT: br bb4([[ZERO]] : $Builtin.Word) +// CHECK: bb4([[IDX:%.*]] : $Builtin.Word) +// CHECK-NEXT: [[IDX_EQ_LEN:%.*]] = builtin "cmp_eq_Word"([[IDX]] : $Builtin.Word, [[LEN]] : $Builtin.Word) : $Builtin.Int1 +// CHECK-NEXT: cond_br [[IDX_EQ_LEN]], bb6, bb5 +// CHECK: bb5: +// FIXME: the pack type on this seems to be a lowered type +// CHECK-NEXT: [[EXPANSION_INDEX:%.*]] = dynamic_pack_index [[IDX]] of +// CHECK-NEXT: open_pack_element [[EXPANSION_INDEX]] of at , shape $each T, uuid [[UUID:".*"]] +// CHECK-NEXT: [[SRC_ELT_ADDR:%.*]] = tuple_pack_element_addr [[EXPANSION_INDEX]] of [[EXPANSION_TEMP]] : $*(repeat @callee_guaranteed @substituted <τ_0_0, τ_0_1> (@in_guaranteed τ_0_0) -> @out τ_0_1 for ) as $*@callee_guaranteed @substituted <τ_0_0, τ_0_1> (@in_guaranteed τ_0_0) -> @out τ_0_1 for <@pack_element([[UUID]]) each T, Bool> +// - Load the function value and apply the thunking conversion. +// CHECK-NEXT: [[ELT:%.*]] = load [take] [[SRC_ELT_ADDR]] : +// CHECK-NEXT: [[DEST_ELT_ADDR:%.*]] = pack_element_get [[EXPANSION_INDEX]] of %0 : $*Pack{repeat @callee_guaranteed @substituted <τ_0_0> (@in_guaranteed τ_0_0) -> Bool for } as $*@callee_guaranteed @substituted <τ_0_0> (@in_guaranteed τ_0_0) -> Bool for <@pack_element([[UUID]]) each T> +// CHECK-NEXT: [[ESCAPING_ELT:%.*]] = convert_function [[ELT]] +// CHECK-NEXT: // function_ref +// CHECK-NEXT: [[THUNK:%.*]] = function_ref @$sxSbIegnr_xSbIegnd_lTR : $@convention(thin) <τ_0_0> (@in_guaranteed τ_0_0, @guaranteed @callee_guaranteed (@in_guaranteed τ_0_0) -> @out Bool) -> Bool +// CHECK-NEXT: [[CONVERTED_ELT:%.*]] = partial_apply +// CHECK-NEXT: [[CONVERTED_ELT_2:%.*]] = convert_function [[CONVERTED_ELT]] +// CHECK-NEXT: store [[CONVERTED_ELT_2]] to [init] [[DEST_ELT_ADDR]] : +// CHECK-NEXT: [[NEXT_IDX:%.*]] = builtin "add_Word"([[IDX]] : $Builtin.Word, [[ONE]] : $Builtin.Word) : $Builtin.Word +// CHECK-NEXT: br bb4([[NEXT_IDX]] : $Builtin.Word) +// CHECK: bb6: +// - Load the last direct component. +// CHECK-NEXT: [[INT:%.*]] = load [trivial] [[INT_TEMP]] : $*Int +// Tuple up the direct components. +// CHECK-NEXT: [[TUPLE:%.*]] = tuple ([[STRING]] : $String, [[INT]] : $Int) +// CHECK-NEXT: dealloc_stack [[INT_TEMP]] : +// CHECK-NEXT: dealloc_stack [[EXPANSION_TEMP]] : +// CHECK-NEXT: dealloc_stack [[STRING_TEMP]] : +// CHECK-NEXT: dealloc_pack [[PACK]] : +// CHECK-NEXT: return [[TUPLE]]