diff --git a/SwiftCompilerSources/Sources/Optimizer/DataStructures/Set.swift b/SwiftCompilerSources/Sources/Optimizer/DataStructures/Set.swift index d2ab37357656e..84ba0e0be71e7 100644 --- a/SwiftCompilerSources/Sources/Optimizer/DataStructures/Set.swift +++ b/SwiftCompilerSources/Sources/Optimizer/DataStructures/Set.swift @@ -228,3 +228,16 @@ struct OperandSet : IntrusiveSet { context.freeOperandSet(bridged) } } + +extension IntrusiveSet { + mutating func insert(contentsOf source: some Sequence) { + for element in source { + _ = insert(element) + } + } + + init(insertContentsOf source: some Sequence, _ context: some Context) { + self.init(context) + insert(contentsOf: source) + } +} diff --git a/SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifyBranch.swift b/SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifyBranch.swift index a138238cefcf5..f424388e8e7c8 100644 --- a/SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifyBranch.swift +++ b/SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifyBranch.swift @@ -49,7 +49,15 @@ private extension BranchInst { let parentBB = parentBlock for (argIdx, op) in operands.enumerated() { - targetBB.arguments[argIdx].uses.replaceAll(with: op.value, context) + let arg = targetBB.arguments[argIdx] + if let phi = Phi(arg), + let bfi = phi.borrowedFrom + { + bfi.uses.replaceAll(with: op.value, context) + context.erase(instruction: bfi) + } else { + arg.uses.replaceAll(with: op.value, context) + } } targetBB.eraseAllArguments(context) diff --git a/SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifySwitchEnum.swift b/SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifySwitchEnum.swift index ccdee34a0cdb8..2eebfdae9f9f7 100644 --- a/SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifySwitchEnum.swift +++ b/SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifySwitchEnum.swift @@ -48,12 +48,14 @@ extension SwitchEnumInst : OnoneSimplifyable { precondition(enumInst.payload == nil || !parentFunction.hasOwnership, "missing payload argument in switch_enum case block") builder.createBranch(to: caseBlock) + context.erase(instruction: self) case 1: builder.createBranch(to: caseBlock, arguments: [enumInst.payload!]) + context.erase(instruction: self) + updateBorrowedFrom(for: [Phi(caseBlock.arguments[0])!], context) default: fatalError("case block of switch_enum cannot have more than 1 argument") } - context.erase(instruction: self) if canEraseEnumInst { context.erase(instructionIncludingDebugUses: enumInst) diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift index 7d28d3247d447..4acc6782bd28a 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift @@ -117,6 +117,7 @@ private func registerSwiftPasses() { registerPass(rangeDumper, { rangeDumper.run($0) }) registerPass(runUnitTests, { runUnitTests.run($0) }) registerPass(testInstructionIteration, { testInstructionIteration.run($0) }) + registerPass(updateBorrowedFromPass, { updateBorrowedFromPass.run($0) }) } private func registerSwiftAnalyses() { @@ -126,4 +127,5 @@ private func registerSwiftAnalyses() { private func registerUtilities() { registerVerifier() + registerBorrowedFromUpdater() } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/BorrowUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/BorrowUtils.swift index 2726a1bdc26a0..c6cab936f8a5d 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/BorrowUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/BorrowUtils.swift @@ -293,7 +293,8 @@ enum BeginBorrowValue { switch value { case let bbi as BeginBorrowInst: self = .beginBorrow(bbi) case let lbi as LoadBorrowInst: self = .loadBorrow(lbi) - case let arg as FunctionArgument: self = .functionArgument(arg) + case let arg as FunctionArgument where arg.ownership == .guaranteed: + self = .functionArgument(arg) case let arg as Argument where arg.isReborrow: self = .reborrow(Phi(arg)!) default: @@ -375,40 +376,14 @@ enum BeginBorrowValue { case let .beginApply(value): return (value.definingInstruction as! BeginApplyInst).token.uses.endingLifetime + case let .reborrow(phi): + return phi.value.lookThroughBorrowedFromUser.uses.endingLifetime default: return value.uses.endingLifetime } } } -/// Find the borrow introducers for `value`. This gives you a set of -/// OSSA lifetimes that directly include `value`. If `value` is owned, -/// or introduces a borrow scope, then `value` is the single -/// introducer for itself. -/// -/// If `value` is an address or any trivial type, then it has no introducers. -/// -/// Example: // introducers: -/// // ~~~~~~~~~~~~ -/// bb0(%0 : @owned $Class, // %0 -/// %1 : @guaranteed $Class): // %1 -/// %borrow0 = begin_borrow %0 // %borrow0 -/// %pair = struct $Pair(%borrow0, %1) // %borrow0, %1 -/// %first = struct_extract %pair // %borrow0, %1 -/// %field = ref_element_addr %first // (none) -/// %load = load_borrow %field : $*C // %load -func gatherBorrowIntroducers(for value: Value, - in borrowIntroducers: inout Stack, - _ context: Context) { - assert(value.ownership == .guaranteed) - - // Cache introducers across multiple instances of BorrowIntroducers. - var cache = BorrowIntroducers.Cache(context) - defer { cache.deinitialize() } - BorrowIntroducers.gather(for: value, in: &borrowIntroducers, - &cache, context) -} - /// Compute the live range for the borrow scopes of a guaranteed value. This returns a separate instruction range for /// each of the value's borrow introducers. /// @@ -418,12 +393,9 @@ func computeBorrowLiveRange(for value: Value, _ context: FunctionPassContext) assert(value.ownership == .guaranteed) var ranges = SingleInlineArray<(BeginBorrowValue, InstructionRange)>() - var introducers = Stack(context) - defer { introducers.deinitialize() } - gatherBorrowIntroducers(for: value, in: &introducers, context) // If introducers is empty, then the dependence is on a trivial value, so // there is no ownership range. - while let beginBorrow = introducers.pop() { + for beginBorrow in value.getBorrowIntroducers(context) { /// FIXME: Remove calls to computeKnownLiveness() as soon as lifetime completion runs immediately after /// SILGen. Instead, this should compute linear liveness for borrowed value by switching over BeginBorrowValue, just /// like LifetimeDependenc.Scope.computeRange(). @@ -432,473 +404,211 @@ func computeBorrowLiveRange(for value: Value, _ context: FunctionPassContext) return ranges } -private struct BorrowIntroducers { - typealias CachedIntroducers = SingleInlineArray - struct Cache { - // Cache the introducers already found for each SILValue. - var valueIntroducers: Dictionary - // Record recursively followed phis to avoid infinite cycles. - // Phis are removed from this set when they are cached. - var pendingPhis: ValueSet - - init(_ context: Context) { - valueIntroducers = Dictionary() - pendingPhis = ValueSet(context) - } - - mutating func deinitialize() { - pendingPhis.deinitialize() +extension Value { + var lookThroughBorrowedFrom: Value { + if let bfi = self as? BorrowedFromInst { + return bfi.borrowedValue.lookThroughBorrowedFrom } + return self } - - let context: Context - // BorrowIntroducers instances are recursively nested in order to - // find outer adjacent phis. Each instance populates a separate - // 'introducers' set. The same value may occur in 'introducers' at - // multiple levels. Each instance, therefore, needs a separate - // introducer set to avoid adding duplicates. - var visitedIntroducers: Set = Set() - - static func gather(for value: Value, in introducers: inout Stack, - _ cache: inout Cache, _ context: Context) { - var borrowIntroducers = BorrowIntroducers(context: context) - borrowIntroducers.gather(for: value, in: &introducers, &cache) - } +} - private mutating func push(_ beginBorrow: BeginBorrowValue, - in introducers: inout Stack) { - if visitedIntroducers.insert(beginBorrow.value.hashable).inserted { - introducers.push(beginBorrow) - } - } +struct BorrowIntroducers : CollectionLikeSequence { + let initialValue: Value + let context: Ctxt - private mutating func push(contentsOf other: S, - in introducers: inout Stack) where S.Element == BeginBorrowValue { - for elem in other { - push(elem, in: &introducers) - } + func makeIterator() -> EnclosingValueIterator { + EnclosingValueIterator(forBorrowIntroducers: initialValue, context) } +} - // This is the identity function (i.e. just adds `value` to `introducers`) - // when: - // - `value` is owned - // - `value` introduces a borrow scope (begin_borrow, load_borrow, reborrow) - // - // Otherwise recurse up the use-def chain to find all introducers. - private mutating func gather(for value: Value, - in introducers: inout Stack, - _ cache: inout Cache) { - assert(value.ownership == .guaranteed) - // Check if this value's introducers have already been added to - // 'introducers' to avoid duplicates and avoid exponential - // recursion on aggregates. - if let cachedIntroducers = cache.valueIntroducers[value.hashable] { - push(contentsOf: cachedIntroducers, in: &introducers) - return - } - introducers.withMarker( - pushElements: { introducers in - gatherUncached(for: value, in: &introducers, &cache) - }, - withNewElements: { newIntroducers in - { cachedIntroducers in - newIntroducers.forEach { cachedIntroducers.push($0) } - }(&cache.valueIntroducers[value.hashable, default: CachedIntroducers()]) - }) +struct EnclosingValues : CollectionLikeSequence { + let initialValue: Value + let context: Ctxt + + func makeIterator() -> EnclosingValueIterator { + EnclosingValueIterator(forEnclosingValues: initialValue, context) } +} - private mutating func gatherUncached(for value: Value, - in introducers: inout Stack, - _ cache: inout Cache) { - // BeginBorrowedValue handles the initial scope introducers: begin_borrow, - // load_borrow, & reborrow. - if let beginBorrow = BeginBorrowValue(value) { - push(beginBorrow, in: &introducers) - return - } - // Handle guaranteed forwarding phis - if let phi = Phi(value) { - gather(forPhi: phi, in: &introducers, &cache) - return - } - // Recurse through guaranteed forwarding non-phi instructions. - guard let forwardingInst = value.forwardingInstruction else { - fatalError("guaranteed value must be forwarding") - } - for operand in forwardingInst.forwardedOperands { - if operand.value.ownership == .guaranteed { - gather(for: operand.value, in: &introducers, &cache); - } - } +// This iterator must be a class because we need a deinit. +// It shouldn't be a performance problem because the optimizer should always be able to stack promote the iterator. +// TODO: Make it a struct once this is possible with non-copyable types. +final class EnclosingValueIterator : IteratorProtocol { + var worklist: ValueWorklist + + init(forBorrowIntroducers value: Value, _ context: some Context) { + self.worklist = ValueWorklist(context) + self.worklist.pushIfNotVisited(value) } - // Find the introducers of a guaranteed forwarding phi's borrow - // scope. The introducers are either dominating values or reborrows - // in the same block as the forwarding phi. - // - // Recurse along the use-def phi web until a begin_borrow is reached. At each - // level, find the outer-adjacent phi, if one exists, otherwise return the - // dominating definition. - // - // Example: - // - // bb1(%reborrow_1 : @reborrow) - // %field = struct_extract %reborrow_1 - // br bb2(%reborrow_1, %field) - // bb2(%reborrow_2 : @reborrow, %forward_2 : @guaranteed) - // end_borrow %reborrow_2 - // - // Calling `gather(forPhi: %forward_2)` - // recursively computes these introducers: - // - // %field is the only value incoming to %forward_2. - // - // %field is introduced by %reborrow_1 via - // gather(for: %field). - // - // %reborrow_1 is remapped to %reborrow_2 in bb2 via - // mapToPhi(bb1, %reborrow_1)). - // - // %reborrow_2 is returned. - // - private mutating func gather(forPhi phi: Phi, - in introducers: inout Stack, - _ cache: inout Cache) { - // Phi cycles are skipped. They cannot contribute any new introducer. - if !cache.pendingPhis.insert(phi.value) { + init(forEnclosingValues value: Value, _ context: some Context) { + self.worklist = ValueWorklist(context) + if value is Undef || value.ownership != .guaranteed { return } - for (pred, value) in zip(phi.predecessors, phi.incomingValues) { - switch value.ownership { - case .none: - continue - case .owned, .unowned: - fatalError("unexpected ownership for a guaranteed phi operand") - case .guaranteed: + if let beginBorrow = BeginBorrowValue(value.lookThroughBorrowedFrom) { + switch beginBorrow { + case let .beginBorrow(bbi): + // Gather the outer enclosing borrow scope. + worklist.pushIfNotVisited(bbi.borrowedValue) + case .loadBorrow, .beginApply, .functionArgument: + // There is no enclosing value on this path. break + case .reborrow(let phi): + worklist.pushIfNotVisited(contentsOf: phi.borrowedFrom!.enclosingValues) } - // Each phi operand requires a new introducer list and visited - // values set. These values will be remapped to successor phis - // before adding them to the caller's introducer list. It may be - // necessary to revisit a value that was already visited by the - // caller before remapping to phis. - var incomingIntroducers = Stack(context) - defer { - incomingIntroducers.deinitialize() - } - BorrowIntroducers.gather(for: value, in: &incomingIntroducers, - &cache, context) - // Map the incoming introducers to an outer-adjacent phi if one exists. - push(contentsOf: mapToGuaranteedPhi(predecessor: pred, - incomingBorrows: incomingIntroducers), - in: &introducers) - } - // Remove this phi from the pending set. This phi may be visited - // again at a different level of phi recursion. In that case, we - // should return the cached introducers so that they can be - // remapped. - cache.pendingPhis.erase(phi.value) - } -} - -// Given incoming borrows on a predecessor path, return the -// corresponding borrows on the successor block. Each incoming borrow is -// either used by a phi in the successor block, or it must dominate -// the successor block. -private func mapToGuaranteedPhi> ( - predecessor: BasicBlock, incomingBorrows: PredecessorSequence) --> LazyMapSequence { - - let branch = predecessor.terminator as! BranchInst - // Gather the new introducers for the successor block. - return incomingBorrows.lazy.map { incomingBorrow in - // Find an outer adjacent phi in the successor block. - let incomingValue = incomingBorrow.value - if let incomingOp = branch.operands.first(where: { $0.value == incomingValue }) { - return BeginBorrowValue(branch.getArgument(for: incomingOp))! + } else { + // Handle forwarded guaranteed values. + worklist.pushIfNotVisited(value) } - // No candidates phi are outer-adjacent phis. The incoming - // `predDef` must dominate the current guaranteed phi. - return incomingBorrow } -} -// Given incoming values on a predecessor path, return the corresponding values on the successor block. Each incoming -// value is either used by a phi in the successor block, or it must dominate the successor block. -// -// This is Logically the same as mapToGuaranteedPhi but more efficient to simply duplicate the code. -private func mapToPhi> ( - predecessor: BasicBlock, incomingValues: PredecessorSequence) - -> LazyMapSequence { - - let branch = predecessor.terminator as! BranchInst - // Gather the new introducers for the successor block. - return incomingValues.lazy.map { incomingValue in - // Find an outer adjacent phi in the successor block. - if let incomingOp = - branch.operands.first(where: { $0.value == incomingValue }) { - return branch.getArgument(for: incomingOp) - } - // No candidates phi are outer-adjacent phis. The incoming - // `predDef` must dominate the current guaranteed phi. - return incomingValue + deinit { + worklist.deinitialize() } -} -/// Find each "enclosing value" whose OSSA lifetime immediately -/// encloses a guaranteed value. The guaranteed `value` being enclosed -/// effectively keeps these enclosing values alive. This lets you walk -/// up the levels of nested OSSA lifetimes to determine all the -/// lifetimes that are kept alive by a given SILValue. In particular, -/// it discovers "outer-adjacent phis": phis that are kept alive by -/// uses of another phi in the same block. -/// -/// If `value` is a forwarded guaranteed value, then this finds the -/// introducers of the current borrow scope, which is never an empty -/// set. -/// -/// If `value` introduces a borrow scope, then this finds the -/// introducers of the outer enclosing borrow scope that contains this -/// inner scope. -/// -/// If `value` is a `begin_borrow`, then this returns its owned operand, or the introducers of its guaranteed operand. -/// -/// If `value` is an owned value, a function argument, or a -/// load_borrow, then this is an empty set. -/// -/// If `value` is a reborrow, then this either returns a dominating -/// enclosing value or an outer adjacent phi. -/// -/// Example: // enclosing value: -/// // ~~~~~~~~~~~~ -/// bb0(%0 : @owned $Class, // (none) -/// %1 : @guaranteed $Class): // (none) -/// %borrow0 = begin_borrow %0 // %0 -/// %pair = struct $Pair(%borrow0, %1) // %borrow0, %1 -/// %first = struct_extract %pair // %borrow0, %1 -/// %field = ref_element_addr %first // (none) -/// %load = load_borrow %field : $*C // %load -/// -/// Example: // enclosing value: -/// // ~~~~~~~~~~~~ -/// %outerBorrow = begin_borrow %0 // %0 -/// %innerBorrow = begin_borrow %outerBorrow // %outerBorrow -/// br bb1(%outerBorrow, %innerBorrow) -/// bb1(%outerReborrow : @reborrow, // %0 -/// %innerReborrow : @reborrow) // %outerReborrow -/// -func gatherEnclosingValues(for value: Value, - in enclosingValues: inout Stack, - _ context: some Context) { + func next() -> Value? { + while let value = worklist.pop() { + switch value.ownership { + case .none, .unowned: + break - var cache = EnclosingValues.Cache(context) - defer { cache.deinitialize() } - EnclosingValues.gather(for: value, in: &enclosingValues, &cache, context) -} + case .owned: + return value -/// Find inner adjacent phis in the same block as `enclosingPhi`. -/// These keep the enclosing (outer adjacent) phi alive. -func gatherInnerAdjacentPhis(for enclosingPhi: Phi, - in innerAdjacentPhis: inout Stack, - _ context: Context) { - for candidatePhi in enclosingPhi.successor.arguments { - var enclosingValues = Stack(context) - defer { enclosingValues.deinitialize() } - gatherEnclosingValues(for: candidatePhi, in: &enclosingValues, context) - if enclosingValues.contains(where: { $0 == enclosingPhi.value}) { - innerAdjacentPhis.push(Phi(candidatePhi)!) + case .guaranteed: + if BeginBorrowValue(value) != nil { + return value + } else if let bfi = value as? BorrowedFromInst { + if bfi.borrowedPhi.isReborrow { + worklist.pushIfNotVisited(bfi.borrowedValue) + } else { + worklist.pushIfNotVisited(contentsOf: bfi.enclosingValues) + } + } else if let forwardingInst = value.forwardingInstruction { + // Recurse through guaranteed forwarding non-phi instructions. + let ops = forwardingInst.forwardedOperands + worklist.pushIfNotVisited(contentsOf: ops.lazy.map { $0.value }) + } else { + fatalError("cannot get borrow introducers for unknown guaranteed value") + } + } } + return nil } } -// Find the enclosing values for any value, including reborrows. -private struct EnclosingValues { - typealias CachedEnclosingValues = SingleInlineArray - struct Cache { - // Cache the enclosing values already found for each Reborrow. - var reborrowToEnclosingValues: Dictionary - // Record recursively followed reborrows to avoid infinite cycles. - // Reborrows are removed from this set when they are cached. - var pendingReborrows: ValueSet - - var borrowIntroducerCache: BorrowIntroducers.Cache - - init(_ context: Context) { - reborrowToEnclosingValues = - Dictionary() - pendingReborrows = ValueSet(context) - borrowIntroducerCache = BorrowIntroducers.Cache(context) - } - - mutating func deinitialize() { - pendingReborrows.deinitialize() - borrowIntroducerCache.deinitialize() - } - } - - var context: Context - // EnclosingValues instances are recursively nested in order to - // find outer adjacent phis. Each instance populates a separate - // 'enclosingValeus' set. The same value may occur in 'enclosingValues' at - // multiple levels. Each instance, therefore, needs a separate - // visited set to avoid adding duplicates. - var visitedEnclosingValues: Set = Set() - - static func gather(for value: Value, in enclosingValues: inout Stack, - _ cache: inout Cache, _ context: Context) { - var gatherValues = EnclosingValues(context: context) - gatherValues.gather(for: value, in: &enclosingValues, &cache) - } - private mutating func push(_ enclosingValue: Value, - in enclosingValues: inout Stack) { - if visitedEnclosingValues.insert(enclosingValue.hashable).inserted { - enclosingValues.push(enclosingValue) - } +extension Value { + /// Get the borrow introducers for this value. This gives you a set of + /// OSSA lifetimes that directly include this value. If this value is owned, + /// or introduces a borrow scope, then this value is the single introducer for itself. + /// + /// If this value is an address or any trivial type, then it has no introducers. + /// + /// Example: // introducers: + /// // ~~~~~~~~~~~~ + /// bb0(%0 : @owned $Class, // %0 + /// %1 : @guaranteed $Class): // %1 + /// %borrow0 = begin_borrow %0 // %borrow0 + /// %pair = struct $Pair(%borrow0, %1) // %borrow0, %1 + /// %first = struct_extract %pair // %borrow0, %1 + /// %field = ref_element_addr %first // (none) + /// %load = load_borrow %field : $*C // %load + /// + func getBorrowIntroducers(_ context: Ctxt) -> LazyMapSequence, BeginBorrowValue> { + BorrowIntroducers(initialValue: self, context: context).lazy.map { BeginBorrowValue($0)! } } - private mutating func push(contentsOf other: S, - in enclosingValues: inout Stack) where S.Element == Value { - for elem in other { - push(elem, in: &enclosingValues) - } + /// Get "enclosing values" whose OSSA lifetime immediately encloses a guaranteed value. + /// + /// The guaranteed value being enclosed effectively keeps these enclosing values alive. + /// This lets you walk up the levels of nested OSSA lifetimes to determine all the + /// lifetimes that are kept alive by a given SILValue. In particular, it discovers "outer-adjacent phis": + /// phis that are kept alive by uses of another phi in the same block. + /// + /// If this value is a forwarded guaranteed value, then this finds the + /// introducers of the current borrow scope, which is never an empty set. + /// + /// If this value introduces a borrow scope, then this finds the introducers of the outer + /// enclosing borrow scope that contains this inner scope. + /// + /// If this value is a `begin_borrow`, then this function returns its operand. + /// + /// If this value is an owned value, a function argument, or a load_borrow, then this is an empty set. + /// + /// If this value is a reborrow, then this either returns a dominating enclosing value or an outer adjacent phi. + /// + /// Example: // enclosing value: + /// // ~~~~~~~~~~~~ + /// bb0(%0 : @owned $Class, // (none) + /// %1 : @guaranteed $Class): // (none) + /// %borrow0 = begin_borrow %0 // %0 + /// %pair = struct $Pair(%borrow0, %1) // %borrow0, %1 + /// %first = struct_extract %pair // %borrow0, %1 + /// %field = ref_element_addr %first // (none) + /// %load = load_borrow %field : $*C // %load + /// + /// Example: // enclosing value: + /// // ~~~~~~~~~~~~ + /// %outerBorrow = begin_borrow %0 // %0 + /// %innerBorrow = begin_borrow %outerBorrow // %outerBorrow + /// br bb1(%outerBorrow, %innerBorrow) + /// bb1(%outerReborrow : @reborrow, // %0 + /// %innerReborrow : @reborrow) // %outerReborrow + /// + func getEnclosingValues(_ context: Ctxt) -> EnclosingValues { + EnclosingValues(initialValue: self, context: context) } +} - mutating func gather(for value: Value, - in enclosingValues: inout Stack, - _ cache: inout Cache) { - if value is Undef || value.ownership != .guaranteed { - return - } - if let beginBorrow = BeginBorrowValue(value) { - switch beginBorrow { - case let .beginBorrow(bbi): - let outerValue = bbi.operand.value - switch outerValue.ownership { - case .none, .unowned: - return - case .owned: - push(outerValue, in: &enclosingValues); - return - case .guaranteed: - break - } - // Gather the outer enclosing borrow scope. - gatherBorrows(for: outerValue, in: &enclosingValues, &cache) - case .loadBorrow, .beginApply, .functionArgument: - // There is no enclosing value on this path. - break - case let .reborrow(reborrow): - gather(forReborrow: reborrow, in: &enclosingValues, &cache) +extension Phi { + /// The inner adjacent phis of this outer "enclosing" phi. + /// These keep the enclosing (outer adjacent) phi alive. + var innerAdjacentPhis: LazyMapSequence>, Phi> { + value.uses.lazy.compactMap { use in + if let bfi = use.instruction as? BorrowedFromInst, + use.index != 0 + { + return Phi(bfi.borrowedValue) } - } else { - // Handle forwarded guaranteed values. - gatherBorrows(for: value, in: &enclosingValues, &cache) + return nil } } +} - mutating func gatherBorrows(for value: Value, in enclosingValues: inout Stack, _ cache: inout Cache) { - var introducers = Stack(context) - defer { introducers.deinitialize() } - BorrowIntroducers.gather(for: value, in: &introducers, &cache.borrowIntroducerCache, context) - for beginBorrow in introducers { - enclosingValues.push(beginBorrow.value) +/// Gathers enclosing values by visiting predecessor blocks. +/// Only used for updating borrowed-from instructions and for verification. +func gatherEnclosingValuesFromPredecessors( + for phi: Phi, + in enclosingValues: inout Stack, + _ context: some Context +) { + var alreadyAdded = ValueSet(context) + defer { alreadyAdded.deinitialize() } + + for predecessor in phi.predecessors { + let incomingOperand = phi.incomingOperand(inPredecessor: predecessor) + + for predEV in incomingOperand.value.getEnclosingValues(context) { + let ev = predecessor.mapToPhiInSuccessor(incomingEnclosingValue: predEV) + if alreadyAdded.insert(ev) { + enclosingValues.push(ev) + } } } +} - // Given a reborrow, find the enclosing values. Each enclosing value - // is represented by one of the following cases, which refer to the - // example below: - // - // dominating owned value -> %value encloses %reborrow_1 - // owned outer-adjacent phi -> %phi_3 encloses %reborrow_3 - // dominating outer borrow introducer -> %outerBorrowB encloses %reborrow - // outer-adjacent reborrow -> %outerReborrow encloses %reborrow - // - // Recurse along the use-def phi web until a begin_borrow is - // reached. Then find all introducers of the begin_borrow's - // operand. At each level, find the outer adjacent phi, if one - // exists, otherwise return the most recently found dominating - // definition. - // - // If `reborrow` was already encountered because of a phi cycle, - // then no enclosingDefs are added. - // - // Example: - // - // %value = ... - // %borrow = begin_borrow %value - // br one(%borrow) - // one(%reborrow_1 : @reborrow) - // br two(%value, %reborrow_1) - // two(%phi_2 : @owned, %reborrow_2 : @reborrow) - // br three(%value, %reborrow_2) - // three(%phi_3 : @owned, %reborrow_3 : @reborrow) - // end_borrow %reborrow_3 - // destroy_value %phi_3 - // - // gather(forReborrow: %reborrow_3) finds %phi_3 by computing - // enclosing defs in this order - // (inner -> outer): - // - // %reborrow_1 -> %value - // %reborrow_2 -> %phi_2 - // %reborrow_3 -> %phi_3 - // - // Example: - // - // %outerBorrowA = begin_borrow - // %outerBorrowB = begin_borrow - // %struct = struct (%outerBorrowA, outerBorrowB) - // %borrow = begin_borrow %struct - // br one(%outerBorrowA, %borrow) - // one(%outerReborrow : @reborrow, %reborrow : @reborrow) - // - // gather(forReborrow: %reborrow) finds (%outerReborrow, %outerBorrowB). - // - // This implementation mirrors BorrowIntroducers.gather(forPhi:in:). - // The difference is that this performs use-def recursion over - // reborrows rather, and at each step, it finds the enclosing values - // of the reborrow operands rather than the borrow introducers of - // the guaranteed phi. - private mutating func gather(forReborrow reborrow: Phi, - in enclosingValues: inout Stack, - _ cache: inout Cache) { - - // Phi cycles are skipped. They cannot contribute any new introducer. - if !cache.pendingReborrows.insert(reborrow.value) { - return - } - if let cachedEnclosingValues = - cache.reborrowToEnclosingValues[reborrow.value.hashable] { - push(contentsOf: cachedEnclosingValues, in: &enclosingValues) - return +extension BasicBlock { + func mapToPhiInSuccessor(incomingEnclosingValue: Value) -> Value { + let branch = terminator as! BranchInst + if let incomingEV = branch.operands.first(where: { $0.value.lookThroughBorrowedFrom == incomingEnclosingValue }) { + return branch.getArgument(for: incomingEV) } - assert(enclosingValues.isEmpty) - - // Find the enclosing introducer for each reborrow operand, and - // remap it to the enclosing introducer for the successor block. - for (pred, incomingValue) - in zip(reborrow.predecessors, reborrow.incomingValues) { - var incomingEnclosingValues = Stack(context) - defer { - incomingEnclosingValues.deinitialize() - } - EnclosingValues.gather(for: incomingValue, in: &incomingEnclosingValues, - &cache, context) - push(contentsOf: mapToPhi(predecessor: pred, - incomingValues: incomingEnclosingValues), - in: &enclosingValues) - } - { cachedIntroducers in - enclosingValues.forEach { cachedIntroducers.push($0) } - }(&cache.reborrowToEnclosingValues[reborrow.value.hashable, - default: CachedEnclosingValues()]) - - // Remove this reborrow from the pending set. It may be visited - // again at a different level of recursion. - cache.pendingReborrows.erase(reborrow.value) + // No candidates phi are outer-adjacent phis. The incoming + // `predDef` must dominate the current guaranteed phi. + return incomingEnclosingValue } } @@ -907,12 +617,9 @@ let borrowIntroducersTest = FunctionTest("borrow_introducers") { let value = arguments.takeValue() print(function) print("Borrow introducers for: \(value)") - var introducers = Stack(context) - defer { - introducers.deinitialize() + for bi in value.lookThroughBorrowedFromUser.getBorrowIntroducers(context) { + print(bi) } - gatherBorrowIntroducers(for: value, in: &introducers, context) - introducers.forEach { print($0.value) } } let enclosingValuesTest = FunctionTest("enclosing_values") { @@ -924,6 +631,19 @@ let enclosingValuesTest = FunctionTest("enclosing_values") { defer { enclosing.deinitialize() } - gatherEnclosingValues(for: value, in: &enclosing, context) - enclosing.forEach { print($0) } + for ev in value.lookThroughBorrowedFromUser.getEnclosingValues(context) { + print(ev) + } +} + +extension Value { + var lookThroughBorrowedFromUser: Value { + for use in uses { + if let bfi = use.forwardingBorrowedFromUser { + return bfi + } + } + return self + } } + diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/BorrowedFromUpdater.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/BorrowedFromUpdater.swift new file mode 100644 index 0000000000000..dd5a4be1f4fa8 --- /dev/null +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/BorrowedFromUpdater.swift @@ -0,0 +1,132 @@ +//===--- BorrowedFromUpdater.swift ----------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SIL +import OptimizerBridging + +/// Update all borrowed-from instructions in the `function` +func updateBorrowedFrom(in function: Function, _ context: some MutatingContext) { + if !function.hasOwnership { + return + } + var guaranteedPhis = Stack(context) + defer { guaranteedPhis.deinitialize() } + + for block in function.blocks { + for arg in block.arguments { + if let phi = Phi(arg), phi.value.ownership == .guaranteed { + guaranteedPhis.append(phi) + } + } + } + updateBorrowedFrom(for: guaranteedPhis, context) +} + +/// Update borrowed-from instructions for a set of phi arguments. +func updateBorrowedFrom(for phis: some Sequence, _ context: some MutatingContext) { + for phi in phis { + if !phi.value.parentFunction.hasOwnership { + return + } + if phi.value.ownership == .guaranteed { + createBorrowedFrom(for: phi, context) + } + } + + var changed: Bool + repeat { + changed = false + + for phi in phis { + if phi.value.ownership == .guaranteed { + changed = updateBorrowedFrom(for: phi, context) || changed + } + } + } while changed +} + +private func updateBorrowedFrom(for phi: Phi, _ context: some MutatingContext) -> Bool { + var computedEVs = Stack(context) + defer { computedEVs.deinitialize() } + gatherEnclosingValuesFromPredecessors(for: phi, in: &computedEVs, context) + + var changed = false + for use in phi.value.uses { + if let bfi = use.forwardingBorrowedFromUser { + changed = addEnclosingValues(computedEVs, to: bfi, context) || changed + } + } + return changed +} + +private func createBorrowedFrom(for phi: Phi, _ context: some MutatingContext) { + if !phi.value.uses.contains(where: { $0.forwardingBorrowedFromUser != nil }) { + let builder = Builder(atBeginOf: phi.value.parentBlock, context) + let bfi = builder.createBorrowedFrom(borrowedValue: phi.value, enclosingValues: []) + phi.value.uses.ignore(user: bfi).replaceAll(with: bfi, context) + } +} + +private func addEnclosingValues( + _ newEVs: some Sequence, + to borrowedFrom: BorrowedFromInst, + _ context: some MutatingContext) -> Bool +{ + var existingEVs = ValueSet(insertContentsOf: borrowedFrom.enclosingValues, context) + defer { existingEVs.deinitialize() } + + if newEVs.allSatisfy({ existingEVs.contains($0) }) { + return false + } + + var evs = Array(borrowedFrom.enclosingValues) + evs.append(contentsOf: newEVs.lazy.filter { !existingEVs.contains($0) }) + + let builder = Builder(before: borrowedFrom, context) + let newBfi = builder.createBorrowedFrom(borrowedValue: borrowedFrom.borrowedValue, enclosingValues: evs) + borrowedFrom.uses.replaceAll(with: newBfi, context) + context.erase(instruction: borrowedFrom) + return true +} + +func registerBorrowedFromUpdater() { + BridgedUtilities.registerBorrowedFromUpdater( + { (bridgedCtxt: BridgedPassContext, bridgedFunction: BridgedFunction) in + let context = FunctionPassContext(_bridged: bridgedCtxt) + let function = bridgedFunction.function; + updateBorrowedFrom(in: function, context) + }, + { (bridgedCtxt: BridgedPassContext, bridgedPhiArray: BridgedArrayRef) in + let context = FunctionPassContext(_bridged: bridgedCtxt) + var guaranteedPhis = Stack(context) + defer { guaranteedPhis.deinitialize() } + bridgedPhiArray.withElements(ofType: BridgedValue.self) { + for bridgedVal in $0 { + let phi = Phi(bridgedVal.value)! + if phi.value.ownership == .guaranteed { + guaranteedPhis.append(phi) + } + } + } + updateBorrowedFrom(for: guaranteedPhis, context) + } + ) +} + +/// This pass is only used for testing. +/// In the regular pipeline it's not needed because optimization passes must make sure that borrowed-from +/// instructions are updated once the pass finishes. +let updateBorrowedFromPass = FunctionPass(name: "update-borrowed-from") { + (function: Function, context: FunctionPassContext) in + + updateBorrowedFrom(in: function, context) +} diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt index 86b4cb269afe5..f53d3f96a512b 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt @@ -8,6 +8,7 @@ swift_compiler_sources(Optimizer AddressUtils.swift + BorrowedFromUpdater.swift BorrowUtils.swift DiagnosticEngine.swift Devirtualization.swift diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift index ba9f093933373..9c859655c418d 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift @@ -381,15 +381,14 @@ extension LifetimeDependence.Scope { } private init?(guaranteed base: Value, _ context: some Context) { - var introducers = Stack(context) - gatherBorrowIntroducers(for: base, in: &introducers, context) // If introducers is empty, then the dependence is on a trivial value, so // there is no dependence scope. // // TODO: Add a SIL verifier check that a mark_dependence [nonescaping] // base is never a guaranteed phi. - guard let beginBorrow = introducers.pop() else { return nil } - assert(introducers.isEmpty, + var iter = base.getBorrowIntroducers(context).makeIterator() + guard let beginBorrow = iter.next() else { return nil } + assert(iter.next() == nil, "guaranteed phis not allowed when diagnosing lifetime dependence") switch beginBorrow { case .beginBorrow, .loadBorrow: diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index 48aec2f70b467..276dea18aa14e 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -622,7 +622,7 @@ extension InstructionRange { case let endBorrow as EndBorrowInst: self.insert(endBorrow) case let branch as BranchInst: - worklist.pushIfNotVisited(branch.getArgument(for: use)) + worklist.pushIfNotVisited(branch.getArgument(for: use).lookThroughBorrowedFromUser) default: break } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OwnershipLiveness.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OwnershipLiveness.swift index 72c72c7c64abd..0dba0d0487ab2 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OwnershipLiveness.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OwnershipLiveness.swift @@ -64,7 +64,7 @@ func computeLinearLiveness(for definingValue: Value, _ context: Context) var range = InstructionRange(for: definingValue, context) // Compute liveness. - definingValue.uses.endingLifetime.forEach { + definingValue.lookThroughBorrowedFromUser.uses.endingLifetime.forEach { range.insert($0.instruction) } return range @@ -560,10 +560,7 @@ struct InteriorUseWalker { // If the outer value is an owned phi or reborrow, consider inner // adjacent phis part of its lifetime. if let phi = Phi(definingValue), phi.endsLifetime { - var innerPhis = Stack(context) - defer { innerPhis.deinitialize() } - gatherInnerAdjacentPhis(for: phi, in: &innerPhis, context) - let result = innerPhis.walk { innerPhi in + let result = phi.innerAdjacentPhis.walk { innerPhi in if innerPhi.isReborrow { // Inner adjacent reborrows are considered inner borrow scopes. if handleInner(borrowed: innerPhi.value) == .abortWalk { @@ -855,11 +852,8 @@ extension InteriorUseWalker { guard visited.insert(guaranteedPhi.value) else { return .continueWalk } - var enclosingValues = Stack(context) - defer { enclosingValues.deinitialize() } - gatherEnclosingValues(for: guaranteedPhi.value, in: &enclosingValues, - context) - guard enclosingValues.contains(definingValue) else { + let phiValue = guaranteedPhi.value.lookThroughBorrowedFromUser + guard phiValue.getEnclosingValues(functionContext).contains(definingValue) else { // Since definingValue is not an enclosing value, it must be // consumed or reborrowed by some outer adjacent phi in this // block. An outer adjacent phi's uses do not contribute to the diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/Verifier.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/Verifier.swift index eb9bd7b018bd9..4b3b5b13cc60f 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/Verifier.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/Verifier.swift @@ -26,6 +26,9 @@ private func require(_ condition: Bool, _ message: @autoclosure () -> String) { extension Function { func verify(_ context: FunctionPassContext) { for block in blocks { + for arg in block.arguments { + arg.verify(context) + } for inst in block.instructions { inst.checkForwardingConformance() @@ -48,6 +51,44 @@ private extension Instruction { } } +extension BorrowedFromInst : VerifyableInstruction { + func verify(_ context: FunctionPassContext) { + var computedEVs = Stack(context) + defer { computedEVs.deinitialize() } + + guard let phi = Phi(borrowedValue) else { + fatalError("borrowed value of borrowed-from must be a phi: \(self)") + } + gatherEnclosingValuesFromPredecessors(for: phi, in: &computedEVs, context) + + var existingEVs = ValueSet(insertContentsOf: enclosingValues, context) + defer { existingEVs.deinitialize() } + + for computedEV in enclosingValues { + require(existingEVs.contains(computedEV), + "\(computedEV)\n missing in enclosing values of \(self)") + } + } +} + +private extension Argument { + func verify(_ context: FunctionPassContext) { + if let phi = Phi(self), phi.value.ownership == .guaranteed { + var forwardingBorrowedFromFound = false + for use in phi.value.uses { + require(use.instruction is BorrowedFromInst, + "guaranteed phi: \(self)\n has non borrowed-from use: \(use)") + if use.index == 0 { + require(!forwardingBorrowedFromFound, "phi \(self) has multiple forwarding borrowed-from uses") + forwardingBorrowedFromFound = true + } + } + require (forwardingBorrowedFromFound, + "missing forwarding borrowed-from user of guaranteed phi \(phi)") + } + } +} + func registerVerifier() { BridgedUtilities.registerVerifier( { (bridgedCtxt: BridgedPassContext, bridgedFunction: BridgedFunction) in diff --git a/SwiftCompilerSources/Sources/SIL/Argument.swift b/SwiftCompilerSources/Sources/SIL/Argument.swift index fffc9e3ceb525..3a5c16631e329 100644 --- a/SwiftCompilerSources/Sources/SIL/Argument.swift +++ b/SwiftCompilerSources/Sources/SIL/Argument.swift @@ -139,6 +139,15 @@ public struct Phi { value.ownership == .owned || value.isReborrow } + public var borrowedFrom: BorrowedFromInst? { + for use in value.uses { + if let bfi = use.forwardingBorrowedFromUser { + return bfi + } + } + return nil + } + public static func ==(lhs: Phi, rhs: Phi) -> Bool { lhs.value === rhs.value } @@ -148,6 +157,15 @@ public struct Phi { } } +extension Operand { + public var forwardingBorrowedFromUser: BorrowedFromInst? { + if let bfi = instruction as? BorrowedFromInst, index == 0 { + return bfi + } + return nil + } +} + public struct TerminatorResult { public let value: Argument diff --git a/SwiftCompilerSources/Sources/SIL/Builder.swift b/SwiftCompilerSources/Sources/SIL/Builder.swift index 20fd429b60970..ae0e17c33bf03 100644 --- a/SwiftCompilerSources/Sources/SIL/Builder.swift +++ b/SwiftCompilerSources/Sources/SIL/Builder.swift @@ -184,6 +184,13 @@ public struct Builder { return notifyNew(bridged.createBeginBorrow(value.bridged).getAs(BeginBorrowInst.self)) } + public func createBorrowedFrom(borrowedValue: Value, enclosingValues: [Value]) -> BorrowedFromInst { + let bfi = enclosingValues.withBridgedValues { valuesRef in + return bridged.createBorrowedFrom(borrowedValue.bridged, valuesRef) + } + return notifyNew(bfi.getAs(BorrowedFromInst.self)) + } + @discardableResult public func createEndBorrow(of beginBorrow: Value) -> EndBorrowInst { return notifyNew(bridged.createEndBorrow(beginBorrow.bridged).getAs(EndBorrowInst.self)) diff --git a/SwiftCompilerSources/Sources/SIL/ForwardingInstruction.swift b/SwiftCompilerSources/Sources/SIL/ForwardingInstruction.swift index 4a6872a807ee2..bb84739bd9795 100644 --- a/SwiftCompilerSources/Sources/SIL/ForwardingInstruction.swift +++ b/SwiftCompilerSources/Sources/SIL/ForwardingInstruction.swift @@ -441,6 +441,14 @@ extension LinearFunctionExtractInst: ForwardingInstruction { public var canForwardOwnedValues: Bool { true } } +extension BorrowedFromInst: ForwardingInstruction { + public var singleForwardedOperand: Operand? { operands[0] } + public var preservesIdentity: Bool { true } + public var preservesRepresentation: Bool { true } + public var canForwardGuaranteedValues: Bool { true } + public var canForwardOwnedValues: Bool { false } +} + // ----------------------------------------------------------------------------- // ownership transition instructions diff --git a/SwiftCompilerSources/Sources/SIL/Instruction.swift b/SwiftCompilerSources/Sources/SIL/Instruction.swift index 2d06d18835d73..5ecb2dbb0bb12 100644 --- a/SwiftCompilerSources/Sources/SIL/Instruction.swift +++ b/SwiftCompilerSources/Sources/SIL/Instruction.swift @@ -956,6 +956,18 @@ final public class BridgeObjectToRefInst : SingleValueInstruction, UnaryInstruct final public class BridgeObjectToWordInst : SingleValueInstruction, UnaryInstruction {} +final public class BorrowedFromInst : SingleValueInstruction, BorrowIntroducingInstruction { + public var borrowedValue: Value { operands[0].value } + public var borrowedPhi: Phi { Phi(borrowedValue)! } + public var enclosingOperands: OperandArray { + let ops = operands + return ops[1...Elements, Value> { + enclosingOperands.values + } +} + final public class ProjectBoxInst : SingleValueInstruction, UnaryInstruction { public var box: Value { operand.value } public var fieldIndex: Int { bridged.ProjectBoxInst_fieldIndex() } diff --git a/SwiftCompilerSources/Sources/SIL/Operand.swift b/SwiftCompilerSources/Sources/SIL/Operand.swift index d353cf45aef6a..f203f74526a7a 100644 --- a/SwiftCompilerSources/Sources/SIL/Operand.swift +++ b/SwiftCompilerSources/Sources/SIL/Operand.swift @@ -150,6 +150,10 @@ extension Sequence where Element == Operand { self.lazy.filter { !($0.instruction is I) } } + public func ignore(user: Instruction) -> LazyFilterSequence { + self.lazy.filter { !($0.instruction == user) } + } + public func getSingleUser(ofType: I.Type) -> I? { filterUsers(ofType: I.self).singleUse?.instruction as? I } diff --git a/SwiftCompilerSources/Sources/SIL/Registration.swift b/SwiftCompilerSources/Sources/SIL/Registration.swift index c8af083f1af32..d2f7c22023727 100644 --- a/SwiftCompilerSources/Sources/SIL/Registration.swift +++ b/SwiftCompilerSources/Sources/SIL/Registration.swift @@ -201,6 +201,7 @@ public func registerSILClasses() { register(ExtractExecutorInst.self) register(BeginAccessInst.self) register(BeginBorrowInst.self) + register(BorrowedFromInst.self) register(ProjectBoxInst.self) register(ProjectExistentialBoxInst.self) register(CopyValueInst.self) diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/SequenceUtilities.swift b/SwiftCompilerSources/Sources/SIL/Utilities/SequenceUtilities.swift index b97c068a683a6..018a86dd57299 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/SequenceUtilities.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/SequenceUtilities.swift @@ -98,6 +98,8 @@ public extension CollectionLikeSequence { } return singleElement } + + var first: Element? { first(where: { _ in true }) } } // Also make the lazy sequences a CollectionLikeSequence if the underlying sequence is one. diff --git a/docs/SIL.rst b/docs/SIL.rst index a31f9f36aefc9..dcc2f7526b6dd 100644 --- a/docs/SIL.rst +++ b/docs/SIL.rst @@ -4615,6 +4615,38 @@ We require that ``%1`` and ``%0`` have the same type ignoring SILValueCategory. This instruction is only valid in functions in Ownership SSA form. +borrowed from +````````````` + +:: + + sil-instruction ::= 'borrowed' sil-operand 'from' '(' (sil-operand (',' sil-operand)*)? ')' + + bb1(%1 : @owned $T, %2 : @guaranteed $T): + %3 = borrowed %2 : $T from (%1 : $T, %0 : $S) + // %3 has type $T and guaranteed ownership + +Declares from which enclosing values a guaranteed phi argument is borrowed +from. +An enclosing value is either a dominating borrow introducer (``%0``) of the +borrowed operand (``%2``) or an adjacent phi-argument in the same block +(``%1``). +In case of an adjacent phi, all incoming values of the adjacent phi must be +borrow introducers for the corresponding incoming value of the borrowed +operand in all predecessor blocks. + +The borrowed operand (``%2``) must be a guaranteed phi argument and is +forwarded to the instruction result. + +The list of enclosing values (operands after ``from``) can be empty if the +borrowed operand stems from a borrow introducer with no enclosing value, e.g. +a ``load_borrow``. + +Guaranteed phi arguments must not have other users than borrowed-from +instructions. + +This instruction is only valid in functions in Ownership SSA form. + end_lifetime ```````````` diff --git a/include/swift/SIL/InstWrappers.h b/include/swift/SIL/InstWrappers.h index d1997056d8a73..d080ab7f294f2 100644 --- a/include/swift/SIL/InstWrappers.h +++ b/include/swift/SIL/InstWrappers.h @@ -279,6 +279,8 @@ class ForwardingOperation { &forwardingInst->getOperandRef(RefToBridgeObjectInst::ConvertedOperand); case SILInstructionKind::TuplePackExtractInst: return &forwardingInst->getOperandRef(TuplePackExtractInst::TupleOperand); + case SILInstructionKind::BorrowedFromInst: + return &forwardingInst->getOperandRef(0); default: int numRealOperands = forwardingInst->getNumRealOperands(); if (numRealOperands == 0) { diff --git a/include/swift/SIL/OwnershipUtils.h b/include/swift/SIL/OwnershipUtils.h index 1a2c1bc753e35..dc038c8a5bf87 100644 --- a/include/swift/SIL/OwnershipUtils.h +++ b/include/swift/SIL/OwnershipUtils.h @@ -90,6 +90,10 @@ inline bool isForwardingConsume(SILValue value) { // Ownership Def-Use Utilities //===----------------------------------------------------------------------===// +BorrowedFromInst *getBorrowedFromUser(SILValue v); +SILValue lookThroughBorrowedFromUser(SILValue v); +SILValue lookThroughBorrowedFromDef(SILValue v); + /// Whether the specified OSSA-lifetime introducer has a pointer escape. /// /// precondition: \p value introduces an OSSA-lifetime, either a BorrowedValue @@ -254,6 +258,7 @@ class BorrowingOperandKind { enum Kind : uint8_t { Invalid = 0, BeginBorrow, + BorrowedFrom, StoreBorrow, BeginApply, Branch, @@ -279,6 +284,8 @@ class BorrowingOperandKind { return Kind::Invalid; case SILInstructionKind::BeginBorrowInst: return Kind::BeginBorrow; + case SILInstructionKind::BorrowedFromInst: + return Kind::BorrowedFrom; case SILInstructionKind::StoreBorrowInst: return Kind::StoreBorrow; case SILInstructionKind::BeginApplyInst: @@ -399,6 +406,7 @@ struct BorrowingOperand { case BorrowingOperandKind::Invalid: llvm_unreachable("Using invalid case?!"); case BorrowingOperandKind::BeginBorrow: + case BorrowingOperandKind::BorrowedFrom: case BorrowingOperandKind::StoreBorrow: case BorrowingOperandKind::BeginApply: case BorrowingOperandKind::Apply: @@ -431,6 +439,7 @@ struct BorrowingOperand { case BorrowingOperandKind::Invalid: llvm_unreachable("Using invalid case?!"); case BorrowingOperandKind::BeginBorrow: + case BorrowingOperandKind::BorrowedFrom: case BorrowingOperandKind::Branch: return true; case BorrowingOperandKind::StoreBorrow: diff --git a/include/swift/SIL/SILBridging.h b/include/swift/SIL/SILBridging.h index 133c8a291369a..abeabe161563c 100644 --- a/include/swift/SIL/SILBridging.h +++ b/include/swift/SIL/SILBridging.h @@ -1163,6 +1163,8 @@ struct BridgedBuilder{ SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createFunctionRef(BridgedFunction function) const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createCopyValue(BridgedValue op) const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createBeginBorrow(BridgedValue op) const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createBorrowedFrom(BridgedValue borrowedValue, + BridgedValueArray enclosingValues) const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createEndBorrow(BridgedValue op) const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createCopyAddr(BridgedValue from, BridgedValue to, bool takeSource, bool initializeDest) const; diff --git a/include/swift/SIL/SILBridgingImpl.h b/include/swift/SIL/SILBridgingImpl.h index d779cf88a3576..63a9564f9f6ea 100644 --- a/include/swift/SIL/SILBridgingImpl.h +++ b/include/swift/SIL/SILBridgingImpl.h @@ -1575,6 +1575,13 @@ BridgedInstruction BridgedBuilder::createBeginBorrow(BridgedValue op) const { return {unbridged().createBeginBorrow(regularLoc(), op.getSILValue())}; } +BridgedInstruction BridgedBuilder::createBorrowedFrom(BridgedValue borrowedValue, + BridgedValueArray enclosingValues) const { + llvm::SmallVector evs; + return {unbridged().createBorrowedFrom(regularLoc(), borrowedValue.getSILValue(), + enclosingValues.getValues(evs))}; +} + BridgedInstruction BridgedBuilder::createEndBorrow(BridgedValue op) const { return {unbridged().createEndBorrow(regularLoc(), op.getSILValue())}; } diff --git a/include/swift/SIL/SILBuilder.h b/include/swift/SIL/SILBuilder.h index 1d99dd44bdb73..4112737e0e1e5 100644 --- a/include/swift/SIL/SILBuilder.h +++ b/include/swift/SIL/SILBuilder.h @@ -837,6 +837,12 @@ class SILBuilder { hasPointerEscape, fromVarDecl, fixed)); } + BorrowedFromInst *createBorrowedFrom(SILLocation Loc, SILValue borrowedValue, + ArrayRef enclosingValues) { + return insert(BorrowedFromInst::create(getSILDebugLocation(Loc), borrowedValue, + enclosingValues, getModule())); + } + /// Convenience function for creating a load_borrow on non-trivial values and /// load [trivial] on trivial values. Becomes load unqualified in non-ossa /// functions. diff --git a/include/swift/SIL/SILCloner.h b/include/swift/SIL/SILCloner.h index 879a08e86dfad..48f1481ea10f2 100644 --- a/include/swift/SIL/SILCloner.h +++ b/include/swift/SIL/SILCloner.h @@ -1240,6 +1240,21 @@ void SILCloner::visitBeginBorrowInst(BeginBorrowInst *Inst) { Inst->hasPointerEscape(), Inst->isFromVarDecl())); } +template +void SILCloner::visitBorrowedFromInst(BorrowedFromInst *bfi) { + getBuilder().setCurrentDebugScope(getOpScope(bfi->getDebugScope())); + if (!getBuilder().hasOwnership()) { + return recordFoldedValue(bfi, getOpValue(bfi->getBorrowedValue())); + } + + auto enclosingValues = getOpValueArray<8>(bfi->getEnclosingValues()); + recordClonedInstruction(bfi, + getBuilder().createBorrowedFrom( + getOpLocation(bfi->getLoc()), + getOpValue(bfi->getBorrowedValue()), + enclosingValues)); +} + template void SILCloner::visitStoreInst(StoreInst *Inst) { getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope())); diff --git a/include/swift/SIL/SILInstruction.h b/include/swift/SIL/SILInstruction.h index 0fea602b446bb..002b2c25a615b 100644 --- a/include/swift/SIL/SILInstruction.h +++ b/include/swift/SIL/SILInstruction.h @@ -4643,6 +4643,37 @@ class BeginBorrowInst Operand *getSingleNonEndingUse() const; }; +/// BorrowedFromInst - Establishes borrow relations. +class BorrowedFromInst final : public InstructionBaseWithTrailingOperands< + SILInstructionKind::BorrowedFromInst, BorrowedFromInst, + OwnershipForwardingSingleValueInstruction> { + friend SILBuilder; + + /// Because of the storage requirements of BorrowedFromInst, object + /// creation goes through 'create()'. + BorrowedFromInst(SILDebugLocation DebugLoc, ArrayRef operands); + + /// Construct a BorrowedFromInst. + static BorrowedFromInst *create(SILDebugLocation DebugLoc, SILValue borrowedValue, + ArrayRef enclosingValues, SILModule &M); + +public: + + SILValue getBorrowedValue() { + return getAllOperands()[0].get(); + } + + /// The elements referenced by this StructInst. + ArrayRef getEnclosingValueOperands() const { + return getAllOperands().drop_front(); + } + + /// The elements referenced by this StructInst. + OperandValueArrayRef getEnclosingValues() const { + return OperandValueArrayRef(getEnclosingValueOperands()); + } +}; + inline auto BeginBorrowInst::getEndBorrows() const -> EndBorrowRange { return getUsersOfType(); } @@ -11136,6 +11167,7 @@ OwnershipForwardingSingleValueInstruction::classof(SILInstructionKind kind) { case SILInstructionKind::UnconditionalCheckedCastInst: case SILInstructionKind::FunctionExtractIsolationInst: case SILInstructionKind::DropDeinitInst: + case SILInstructionKind::BorrowedFromInst: return true; default: return false; diff --git a/include/swift/SIL/SILNodes.def b/include/swift/SIL/SILNodes.def index 7a00efc4d2ceb..24b1f18b4b69e 100644 --- a/include/swift/SIL/SILNodes.def +++ b/include/swift/SIL/SILNodes.def @@ -543,6 +543,8 @@ ABSTRACT_VALUE_AND_INST(SingleValueInstruction, ValueBase, SILInstruction) SingleValueInstruction, MayRead, DoesNotRelease) SINGLE_VALUE_INST(BeginBorrowInst, begin_borrow, SingleValueInstruction, MayHaveSideEffects, DoesNotRelease) + SINGLE_VALUE_INST(BorrowedFromInst, borrowed, + SingleValueInstruction, None, DoesNotRelease) SINGLE_VALUE_INST(StoreBorrowInst, store_borrow, SILInstruction, MayWrite, DoesNotRelease) // begin_access may trap. Trapping is unordered with respect to memory access, diff --git a/include/swift/SIL/SILValue.h b/include/swift/SIL/SILValue.h index 756e96e95ae58..30434ae03176a 100644 --- a/include/swift/SIL/SILValue.h +++ b/include/swift/SIL/SILValue.h @@ -1073,8 +1073,11 @@ class Operand { removeFromCurrent(); TheValue = newValue; insertIntoCurrent(); + verify(); } + void verify() const; + /// Swap the given operand with the current one. void swap(Operand &Op) { SILValue OtherV = Op.get(); diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index cc0fee835d02e..431ba0419256e 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -131,8 +131,12 @@ struct BridgedPostDomTree { struct BridgedUtilities { typedef void (* _Nonnull VerifyFunctionFn)(BridgedPassContext, BridgedFunction); + typedef void (* _Nonnull UpdateBorrowedFromFn)(BridgedPassContext, BridgedFunction); + typedef void (* _Nonnull UpdateBorrowedFromPhisFn)(BridgedPassContext, BridgedArrayRef); static void registerVerifier(VerifyFunctionFn verifyFunctionFn); + static void registerBorrowedFromUpdater(UpdateBorrowedFromFn updateBorrowedFromFn, + UpdateBorrowedFromPhisFn updateBorrowedFromPhisFn); }; struct BridgedBasicBlockSet { diff --git a/include/swift/SILOptimizer/PassManager/Passes.def b/include/swift/SILOptimizer/PassManager/Passes.def index 59cfc87b8e0c4..2da58e2aa7df4 100644 --- a/include/swift/SILOptimizer/PassManager/Passes.def +++ b/include/swift/SILOptimizer/PassManager/Passes.def @@ -501,6 +501,8 @@ PASS(ReferenceBindingTransform, "sil-reference-binding-transform", "Check/transform reference bindings") PASS(DiagnosticDeadFunctionElimination, "sil-diagnostic-dead-function-elim", "Eliminate dead functions from early specialization optimizations before we run later diagnostics") +SWIFT_FUNCTION_PASS(UpdateBorrowedFrom, "update-borrowed-from", + "Test pass for update borrowed-from instructions") PASS(PruneVTables, "prune-vtables", "Mark class methods that do not require vtable dispatch") PASS_RANGE(AllPasses, AADumper, PruneVTables) diff --git a/include/swift/SILOptimizer/Utils/BasicBlockOptUtils.h b/include/swift/SILOptimizer/Utils/BasicBlockOptUtils.h index e14387c888345..a7a5099c6e8e4 100644 --- a/include/swift/SILOptimizer/Utils/BasicBlockOptUtils.h +++ b/include/swift/SILOptimizer/Utils/BasicBlockOptUtils.h @@ -102,16 +102,6 @@ inline bool isUsedOutsideOfBlock(SILValue v) { return false; } -/// Rotate a loop's header as long as it is exiting and not equal to the -/// passed basic block. -/// If \p RotateSingleBlockLoops is true a single basic block loop will be -/// rotated once. ShouldVerify specifies whether to perform verification after -/// the transformation. -/// Returns true if the loop could be rotated. -bool rotateLoop(SILLoop *loop, DominanceInfo *domInfo, SILLoopInfo *loopInfo, - bool rotateSingleBlockLoops, SILBasicBlock *upToBB, - bool shouldVerify); - //===----------------------------------------------------------------------===// // BasicBlock Cloning //===----------------------------------------------------------------------===// @@ -209,6 +199,8 @@ class BasicBlockCloner : public SILCloner { // If available, the current DeadEndBlocks for incremental update. DeadEndBlocks *deBlocks; + SILPassManager *pm; + public: /// An ordered list of old to new available value pairs. /// @@ -217,8 +209,8 @@ class BasicBlockCloner : public SILCloner { SmallVector, 16> availVals; // Clone blocks starting at `origBB`, within the same function. - BasicBlockCloner(SILBasicBlock *origBB, DeadEndBlocks *deBlocks = nullptr) - : SILCloner(*origBB->getParent()), origBB(origBB), deBlocks(deBlocks) {} + BasicBlockCloner(SILBasicBlock *origBB, SILPassManager *pm, DeadEndBlocks *deBlocks = nullptr) + : SILCloner(*origBB->getParent()), origBB(origBB), deBlocks(deBlocks), pm(pm) {} bool canCloneBlock() { for (auto &inst : *origBB) { diff --git a/include/swift/SILOptimizer/Utils/Devirtualize.h b/include/swift/SILOptimizer/Utils/Devirtualize.h index 6a2f352c9bd25..4b647d35958d3 100644 --- a/include/swift/SILOptimizer/Utils/Devirtualize.h +++ b/include/swift/SILOptimizer/Utils/Devirtualize.h @@ -68,7 +68,7 @@ SubstitutionMap getWitnessMethodSubstitutions(SILModule &Module, ApplySite AI, /// /// Return the new apply and true if the CFG was also modified. std::pair -tryDevirtualizeApply(ApplySite AI, ClassHierarchyAnalysis *CHA, +tryDevirtualizeApply(SILPassManager *pm, ApplySite AI, ClassHierarchyAnalysis *CHA, OptRemark::Emitter *ORE = nullptr, bool isMandatory = false); bool canDevirtualizeApply(FullApplySite AI, ClassHierarchyAnalysis *CHA); @@ -86,7 +86,8 @@ CanType getSelfInstanceType(CanType ClassOrMetatypeType); /// /// Return the new apply and true if the CFG was also modified. std::pair -devirtualizeClassMethod(FullApplySite AI, SILValue ClassInstance, ClassDecl *CD, +devirtualizeClassMethod(SILPassManager *pm, + FullApplySite AI, SILValue ClassInstance, ClassDecl *CD, CanType classType, OptRemark::Emitter *ORE); /// Attempt to devirtualize the given apply site, which is known to be @@ -97,7 +98,8 @@ devirtualizeClassMethod(FullApplySite AI, SILValue ClassInstance, ClassDecl *CD, /// /// Return the new apply and true if the CFG was also modified. std::pair -tryDevirtualizeClassMethod(FullApplySite AI, SILValue ClassInstance, +tryDevirtualizeClassMethod(SILPassManager *pm, + FullApplySite AI, SILValue ClassInstance, ClassDecl *CD, CanType ClassType, OptRemark::Emitter *ORE, bool isEffectivelyFinalMethod = false); @@ -110,7 +112,8 @@ tryDevirtualizeClassMethod(FullApplySite AI, SILValue ClassInstance, /// the original apply site. /// /// Return the new apply and true if the CFG was also modified. -std::pair tryDevirtualizeWitnessMethod(ApplySite AI, +std::pair tryDevirtualizeWitnessMethod(SILPassManager *pm, + ApplySite AI, OptRemark::Emitter *ORE, bool isMandatory); diff --git a/include/swift/SILOptimizer/Utils/InstOptUtils.h b/include/swift/SILOptimizer/Utils/InstOptUtils.h index 6472c89e24449..b978859a1f8dd 100644 --- a/include/swift/SILOptimizer/Utils/InstOptUtils.h +++ b/include/swift/SILOptimizer/Utils/InstOptUtils.h @@ -194,7 +194,8 @@ SILValue getConcreteValueOfExistentialBoxAddr(SILValue addr, /// BranchInst (a phi is never the last guaranteed user). \p builder's current /// insertion point must dominate all \p usePoints. std::pair -castValueToABICompatibleType(SILBuilder *builder, SILLocation Loc, +castValueToABICompatibleType(SILBuilder *builder, SILPassManager *pm, + SILLocation Loc, SILValue value, SILType srcTy, SILType destTy, ArrayRef usePoints); /// Peek through trivial Enum initialization, typically for pointless @@ -233,7 +234,7 @@ SILLinkage getSpecializedLinkage(SILFunction *f, SILLinkage linkage); /// Tries to perform jump-threading on all checked_cast_br instruction in /// function \p Fn. bool tryCheckedCastBrJumpThreading( - SILFunction *fn, DominanceInfo *dt, DeadEndBlocks *deBlocks, + SILFunction *fn, SILPassManager *pm, DominanceInfo *dt, DeadEndBlocks *deBlocks, SmallVectorImpl &blocksForWorklist, bool EnableOSSARewriteTerminator); diff --git a/include/swift/SILOptimizer/Utils/OwnershipOptUtils.h b/include/swift/SILOptimizer/Utils/OwnershipOptUtils.h index 18de48372c95e..6edfff2d3bc3a 100644 --- a/include/swift/SILOptimizer/Utils/OwnershipOptUtils.h +++ b/include/swift/SILOptimizer/Utils/OwnershipOptUtils.h @@ -28,6 +28,8 @@ namespace swift { +class SILPassManager; + /// Returns true if this value requires OSSA cleanups. inline bool requiresOSSACleanup(SILValue v) { return v->getFunction()->hasOwnership() && @@ -355,6 +357,10 @@ bool extendStoreBorrow(StoreBorrowInst *sbi, DeadEndBlocks *deadEndBlocks, InstModCallbacks callbacks = InstModCallbacks()); +void updateBorrowedFrom(SILPassManager *pm, SILFunction *f); + +void updateBorrowedFromPhis(SILPassManager *pm, ArrayRef phis); + } // namespace swift #endif diff --git a/lib/IRGen/IRGenSIL.cpp b/lib/IRGen/IRGenSIL.cpp index f022439b34d5d..c06d7c6a2931f 100644 --- a/lib/IRGen/IRGenSIL.cpp +++ b/lib/IRGen/IRGenSIL.cpp @@ -1339,6 +1339,9 @@ class IRGenSILFunction : void visitBeginBorrowInst(BeginBorrowInst *i) { llvm_unreachable("unimplemented"); } + void visitBorrowedFromInst(BorrowedFromInst *i) { + llvm_unreachable("unimplemented"); + } void visitEndBorrowInst(EndBorrowInst *i) { llvm_unreachable("unimplemented"); } diff --git a/lib/SIL/IR/OperandOwnership.cpp b/lib/SIL/IR/OperandOwnership.cpp index 83e0f6828528b..88fccc0991a57 100644 --- a/lib/SIL/IR/OperandOwnership.cpp +++ b/lib/SIL/IR/OperandOwnership.cpp @@ -421,6 +421,12 @@ OperandOwnershipClassifier::visitBeginBorrowInst(BeginBorrowInst *borrow) { return OperandOwnership::Borrow; } +OperandOwnership +OperandOwnershipClassifier::visitBorrowedFromInst(BorrowedFromInst *bfi) { + return getOperandIndex() == 0 ? OperandOwnership::GuaranteedForwarding + : OperandOwnership::InstantaneousUse; +} + // MARK: Instructions whose use ownership depends on the operand in question. OperandOwnership OperandOwnershipClassifier:: diff --git a/lib/SIL/IR/SILArgument.cpp b/lib/SIL/IR/SILArgument.cpp index 11f362c533a0b..f7e460222435d 100644 --- a/lib/SIL/IR/SILArgument.cpp +++ b/lib/SIL/IR/SILArgument.cpp @@ -16,6 +16,7 @@ #include "swift/SIL/SILFunction.h" #include "swift/SIL/SILInstruction.h" #include "swift/SIL/SILModule.h" +#include "swift/SIL/OwnershipUtils.h" #include "llvm/ADT/STLExtras.h" using namespace swift; @@ -291,7 +292,8 @@ bool SILPhiArgument::visitTransitiveIncomingPhiOperands( argument->getIncomingPhiOperands(operands); for (auto *operand : operands) { - SILPhiArgument *forwarded = dyn_cast(operand->get()); + SILValue opVal = lookThroughBorrowedFromDef(operand->get()); + SILPhiArgument *forwarded = dyn_cast(opVal); if (forwarded && forwarded->isPhi()) { worklist.insert(forwarded); } diff --git a/lib/SIL/IR/SILInstructions.cpp b/lib/SIL/IR/SILInstructions.cpp index f1c8cd7f58ace..e2ee48abeaaf4 100644 --- a/lib/SIL/IR/SILInstructions.cpp +++ b/lib/SIL/IR/SILInstructions.cpp @@ -1477,6 +1477,24 @@ StructInst::StructInst(SILDebugLocation Loc, SILType Ty, assert(!Ty.getStructOrBoundGenericStruct()->hasUnreferenceableStorage()); } +BorrowedFromInst *BorrowedFromInst::create(SILDebugLocation DebugLoc, SILValue borrowedValue, + ArrayRef enclosingValues, SILModule &M) { + auto Size = totalSizeToAlloc(enclosingValues.size() + 1); + auto Buffer = M.allocateInst(Size, alignof(StructInst)); + SmallVector operands; + operands.push_back(borrowedValue); + for (SILValue ev : enclosingValues) { + operands.push_back(ev); + } + return ::new (Buffer) BorrowedFromInst(DebugLoc, operands); +} + +BorrowedFromInst::BorrowedFromInst(SILDebugLocation DebugLoc, ArrayRef operands) + : InstructionBaseWithTrailingOperands(operands, DebugLoc, operands[0]->getType(), + operands[0]->getOwnershipKind()) { + assert(operands[0]->getOwnershipKind() != OwnershipKind::Owned); +} + ObjectInst *ObjectInst::create(SILDebugLocation Loc, SILType Ty, ArrayRef Elements, unsigned NumBaseElements, SILModule &M) { diff --git a/lib/SIL/IR/SILPrinter.cpp b/lib/SIL/IR/SILPrinter.cpp index 6f9a82a72b075..7d2c6590dca19 100644 --- a/lib/SIL/IR/SILPrinter.cpp +++ b/lib/SIL/IR/SILPrinter.cpp @@ -1751,6 +1751,20 @@ class SILPrinter : public SILInstructionVisitor { *this << getIDAndType(BBI->getOperand()); } + void visitBorrowedFromInst(BorrowedFromInst *bfi) { + *this << getIDAndType(bfi->getBorrowedValue()); + *this << " from ("; + bool first = true; + for (SILValue ev : bfi->getEnclosingValues()) { + if (!first) { + *this << ", "; + } + first = false; + *this << getIDAndType(ev); + } + *this << ")"; + } + void printStoreOwnershipQualifier(StoreOwnershipQualifier Qualifier) { switch (Qualifier) { case StoreOwnershipQualifier::Unqualified: diff --git a/lib/SIL/IR/SILValue.cpp b/lib/SIL/IR/SILValue.cpp index 0cb1060c217fe..ff0b9decc4760 100644 --- a/lib/SIL/IR/SILValue.cpp +++ b/lib/SIL/IR/SILValue.cpp @@ -422,6 +422,12 @@ llvm::raw_ostream &swift::operator<<(llvm::raw_ostream &os, // Operand //===----------------------------------------------------------------------===// +void Operand::verify() const { + if (isa(getUser()) && getOperandNumber() == 0) { + assert(isa(get()) || isa(get())); + } +} + SILBasicBlock *Operand::getParentBlock() const { auto *self = const_cast(this); return self->getUser()->getParent(); diff --git a/lib/SIL/IR/ValueOwnership.cpp b/lib/SIL/IR/ValueOwnership.cpp index a574e5bd402ac..2786aa1174117 100644 --- a/lib/SIL/IR/ValueOwnership.cpp +++ b/lib/SIL/IR/ValueOwnership.cpp @@ -72,6 +72,7 @@ CONSTANT_OWNERSHIP_INST(Owned, WeakCopyValue) #include "swift/AST/ReferenceStorage.def" CONSTANT_OWNERSHIP_INST(Guaranteed, BeginBorrow) +CONSTANT_OWNERSHIP_INST(Guaranteed, BorrowedFrom) CONSTANT_OWNERSHIP_INST(Guaranteed, LoadBorrow) CONSTANT_OWNERSHIP_INST(Guaranteed, FunctionExtractIsolation) CONSTANT_OWNERSHIP_INST(None, GlobalValue) diff --git a/lib/SIL/Parser/ParseSIL.cpp b/lib/SIL/Parser/ParseSIL.cpp index 9e1e662902266..6c9fbf496fc8d 100644 --- a/lib/SIL/Parser/ParseSIL.cpp +++ b/lib/SIL/Parser/ParseSIL.cpp @@ -3779,6 +3779,30 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B, break; } + case SILInstructionKind::BorrowedFromInst: { + SILValue guaranteedValue; + if (parseTypedValueRef(guaranteedValue, B)) + return true; + + if (parseVerbatim("from") || + P.parseToken(tok::l_paren, diag::expected_tok_in_sil_instr, "(")) + return true; + + if (P.Tok.isNot(tok::r_paren)) { + do { + if (parseTypedValueRef(Val, B)) + return true; + OpList.push_back(Val); + } while (P.consumeIf(tok::comma)); + } + if (P.parseToken(tok::r_paren, diag::expected_tok_in_sil_instr, ")")) + return true; + + ResultVal = B.createBorrowedFrom(InstLoc, guaranteedValue, OpList); + break; + } + + #define NEVER_OR_SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, ...) \ case SILInstructionKind::Load##Name##Inst: { \ bool isTake = false; \ diff --git a/lib/SIL/Utils/BasicBlockUtils.cpp b/lib/SIL/Utils/BasicBlockUtils.cpp index dd49ff790d1ad..908b0908a8fe7 100644 --- a/lib/SIL/Utils/BasicBlockUtils.cpp +++ b/lib/SIL/Utils/BasicBlockUtils.cpp @@ -16,6 +16,7 @@ #include "swift/Basic/STLExtras.h" #include "swift/SIL/Dominance.h" #include "swift/SIL/LoopInfo.h" +#include "swift/SIL/OwnershipUtils.h" #include "swift/SIL/SILArgument.h" #include "swift/SIL/SILBasicBlock.h" #include "swift/SIL/SILBuilder.h" @@ -345,8 +346,14 @@ void swift::mergeBasicBlockWithSingleSuccessor(SILBasicBlock *BB, // If there are any BB arguments in the destination, replace them with the // branch operands, since they must dominate the dest block. - for (unsigned i = 0, e = BI->getArgs().size(); i != e; ++i) - succBB->getArgument(i)->replaceAllUsesWith(BI->getArg(i)); + for (unsigned i = 0, e = BI->getArgs().size(); i != e; ++i) { + SILArgument *arg = succBB->getArgument(i); + if (auto *bfi = getBorrowedFromUser(arg)) { + bfi->replaceAllUsesWith(arg); + bfi->eraseFromParent(); + } + arg->replaceAllUsesWith(BI->getArg(i)); + } BI->eraseFromParent(); diff --git a/lib/SIL/Utils/InstructionUtils.cpp b/lib/SIL/Utils/InstructionUtils.cpp index 8f3ddfc9c912c..53e48d9fdd1eb 100644 --- a/lib/SIL/Utils/InstructionUtils.cpp +++ b/lib/SIL/Utils/InstructionUtils.cpp @@ -506,6 +506,7 @@ RuntimeEffect swift::getRuntimeEffect(SILInstruction *inst, SILType &impactType) case SILInstructionKind::LoadInst: case SILInstructionKind::LoadBorrowInst: case SILInstructionKind::BeginBorrowInst: + case SILInstructionKind::BorrowedFromInst: case SILInstructionKind::StoreBorrowInst: case SILInstructionKind::MarkUninitializedInst: case SILInstructionKind::ProjectExistentialBoxInst: diff --git a/lib/SIL/Utils/OwnershipUtils.cpp b/lib/SIL/Utils/OwnershipUtils.cpp index 3c8faf8affb58..a59f8ea127e86 100644 --- a/lib/SIL/Utils/OwnershipUtils.cpp +++ b/lib/SIL/Utils/OwnershipUtils.cpp @@ -34,7 +34,7 @@ bool swift::findPointerEscape(SILValue original) { ValueWorklist worklist(original->getFunction()); worklist.push(original); - if (auto *phi = SILArgument::asPhi(original)) { + if (auto *phi = SILArgument::asPhi(lookThroughBorrowedFromDef(original))) { phi->visitTransitiveIncomingPhiOperands([&](auto *phi, auto *operand) { worklist.pushIfNotVisited(operand->get()); return true; @@ -168,6 +168,7 @@ bool swift::computeIsGuaranteedForwarding(SILValue value) { if (value->getOwnershipKind() != OwnershipKind::Guaranteed) { return false; } + value = lookThroughBorrowedFromDef(value); // NOTE: canOpcodeForwardInnerGuaranteedValues returns true for transformation // terminator results. if (canOpcodeForwardInnerGuaranteedValues(value) || @@ -185,9 +186,9 @@ bool swift::computeIsGuaranteedForwarding(SILValue value) { // OwnershipVerifier. bool isGuaranteedForwardingPhi = false; phi->visitTransitiveIncomingPhiOperands([&](auto *, auto *op) -> bool { - auto opValue = op->get(); - assert(opValue->getOwnershipKind().isCompatibleWith( + assert(op->get()->getOwnershipKind().isCompatibleWith( OwnershipKind::Guaranteed)); + auto opValue = lookThroughBorrowedFromDef(op->get()); if (canOpcodeForwardInnerGuaranteedValues(opValue) || isa(opValue)) { isGuaranteedForwardingPhi = true; @@ -202,6 +203,30 @@ bool swift::computeIsGuaranteedForwarding(SILValue value) { return isGuaranteedForwardingPhi; } +BorrowedFromInst *swift::getBorrowedFromUser(SILValue v) { + for (auto *use : v->getUses()) { + if (auto *bfi = dyn_cast(use->getUser())) { + if (use->getOperandNumber() == 0) { + return bfi; + } + } + } + return nullptr; +} + +SILValue swift::lookThroughBorrowedFromUser(SILValue v) { + if (BorrowedFromInst *bfi = getBorrowedFromUser(v)) + return bfi; + return v; +} + +SILValue swift::lookThroughBorrowedFromDef(SILValue v) { + while (auto *bfi = dyn_cast(v)) { + v = bfi->getBorrowedValue(); + } + return v; +} + //===----------------------------------------------------------------------===// // Guaranteed Use-Point (Lifetime) Discovery //===----------------------------------------------------------------------===// @@ -609,6 +634,9 @@ void BorrowingOperandKind::print(llvm::raw_ostream &os) const { case Kind::BeginBorrow: os << "BeginBorrow"; return; + case Kind::BorrowedFrom: + os << "BorrowedFrom"; + return; case Kind::StoreBorrow: os << "StoreBorrow"; return; @@ -664,6 +692,7 @@ bool BorrowingOperand::hasEmptyRequiredEndingUses() const { case BorrowingOperandKind::Invalid: llvm_unreachable("Using invalid case"); case BorrowingOperandKind::BeginBorrow: + case BorrowingOperandKind::BorrowedFrom: case BorrowingOperandKind::StoreBorrow: case BorrowingOperandKind::BeginApply: case BorrowingOperandKind::BeginAsyncLet: @@ -691,9 +720,10 @@ bool BorrowingOperand::visitScopeEndingUses( switch (kind) { case BorrowingOperandKind::Invalid: llvm_unreachable("Using invalid case"); - case BorrowingOperandKind::BeginBorrow: { + case BorrowingOperandKind::BeginBorrow: + case BorrowingOperandKind::BorrowedFrom: { bool deadBorrow = true; - for (auto *use : cast(op->getUser())->getUses()) { + for (auto *use : cast(op->getUser())->getUses()) { if (use->isLifetimeEnding()) { deadBorrow = false; if (!visitScopeEnd(use)) @@ -805,8 +835,9 @@ BorrowedValue BorrowingOperand::getBorrowIntroducingUserResult() const { case BorrowingOperandKind::StoreBorrow: return BorrowedValue(); - case BorrowingOperandKind::BeginBorrow: { - auto value = BorrowedValue(cast(op->getUser())); + case BorrowingOperandKind::BeginBorrow: + case BorrowingOperandKind::BorrowedFrom: { + auto value = BorrowedValue(cast(op->getUser())); assert(value); return value; } @@ -833,6 +864,7 @@ SILValue BorrowingOperand::getScopeIntroducingUserResult() { case BorrowingOperandKind::PartialApplyStack: case BorrowingOperandKind::MarkDependenceNonEscaping: case BorrowingOperandKind::BeginBorrow: + case BorrowingOperandKind::BorrowedFrom: case BorrowingOperandKind::StoreBorrow: return cast(op->getUser()); @@ -901,10 +933,11 @@ void BorrowedValue::getLocalScopeEndingInstructions( // Note: BorrowedLifetimeExtender assumes no intermediate values between a // borrow introducer and its reborrow. The borrowed value must be an operand of -// the reborrow. +// the reborrow. Exception: it looks through `borrowed-from` of a reborrow phi. bool BorrowedValue::visitLocalScopeEndingUses( function_ref visitor) const { assert(isLocalScope() && "Should only call this given a local scope"); + SILValue v = value; switch (kind) { case BorrowedValueKind::Invalid: llvm_unreachable("Using invalid case?!"); @@ -913,7 +946,7 @@ bool BorrowedValue::visitLocalScopeEndingUses( case BorrowedValueKind::LoadBorrow: case BorrowedValueKind::BeginBorrow: case BorrowedValueKind::Phi: - for (auto *use : value->getUses()) { + for (auto *use : lookThroughBorrowedFromUser(value)->getUses()) { if (use->isLifetimeEnding()) { if (!visitor(use)) return false; @@ -1682,7 +1715,7 @@ void swift::visitExtendedGuaranteedForwardingPhiBaseValuePairs( // then set newBaseValue to the reborrow. for (auto &op : phiOp.getBranch()->getAllOperands()) { PhiOperand otherPhiOp(&op); - if (otherPhiOp.getSource() != currentBaseValue) { + if (lookThroughBorrowedFromDef(otherPhiOp.getSource()) != currentBaseValue) { continue; } newBaseValue = otherPhiOp.getValue(); @@ -1979,6 +2012,8 @@ class FindEnclosingDefs { assert(incomingValue->getOwnershipKind() == OwnershipKind::Guaranteed); + incomingValue = lookThroughBorrowedFromDef(incomingValue); + // Avoid repeatedly constructing BorrowedValue during use-def // traversal. That would be quadratic if it checks all uses for reborrows. if (auto *predPhi = dyn_cast(incomingValue)) { @@ -2262,7 +2297,7 @@ void swift::visitTransitiveEndBorrows( while (!worklist.empty()) { auto val = worklist.pop(); - for (auto *consumingUse : val->getConsumingUses()) { + for (auto *consumingUse : lookThroughBorrowedFromUser(val)->getConsumingUses()) { auto *consumingUser = consumingUse->getUser(); if (auto *branch = dyn_cast(consumingUser)) { auto *succBlock = branch->getSingleSuccessorBlock(); diff --git a/lib/SIL/Verifier/GuaranteedPhiVerifier.cpp b/lib/SIL/Verifier/GuaranteedPhiVerifier.cpp index 8b47458c676e5..9c59dd3a1f892 100644 --- a/lib/SIL/Verifier/GuaranteedPhiVerifier.cpp +++ b/lib/SIL/Verifier/GuaranteedPhiVerifier.cpp @@ -28,12 +28,12 @@ bool GuaranteedPhiVerifier::verifyDependentPhiLifetime(SILPhiArgument *phi, dependentPhiToBaseValueMap[phi].insert(baseValue); // Now, verify whether phi's lifetime is within baseValue's lifetime - SmallVector baseValConsumingUses(baseValue->getConsumingUses()); + SmallVector baseValConsumingUses(lookThroughBorrowedFromUser(baseValue)->getConsumingUses()); // If the baseValue has no consuming uses, there is nothing more to verify if (baseValConsumingUses.empty()) return false; - SmallVector phiUses(phi->getUses()); + SmallVector phiUses(lookThroughBorrowedFromUser(phi)->getUses()); LinearLifetimeChecker checker(deadEndBlocks); // newErrorBuilder is consumed at the end of the checkValue function. // Copy initial state from errorBuilder everytime diff --git a/lib/SIL/Verifier/SILOwnershipVerifier.cpp b/lib/SIL/Verifier/SILOwnershipVerifier.cpp index 621e4c525f68a..d3f56b34f32c3 100644 --- a/lib/SIL/Verifier/SILOwnershipVerifier.cpp +++ b/lib/SIL/Verifier/SILOwnershipVerifier.cpp @@ -342,7 +342,8 @@ bool SILValueOwnershipChecker::gatherUsers( // lifetime ending use. Otherwise, we have a guaranteed value that has // an end_borrow on a forwarded value which is not supported in any // case, so emit an error. - if (op->get() != value) { + // The only exception is a `borrowed-from` instruction. + if (op->get() != value && !isa(op->get())) { errorBuilder.handleMalformedSIL([&] { llvm::errs() << "Invalid End Borrow!\n" << "Original Value: " << value @@ -638,7 +639,7 @@ bool SILValueOwnershipChecker:: bool foundGuaranteedForwardingPhiOperand = false; bool foundNonGuaranteedForwardingPhiOperand = false; phi->visitTransitiveIncomingPhiOperands([&](auto *, auto *operand) -> bool { - auto value = operand->get(); + auto value = lookThroughBorrowedFromDef(operand->get()); if (canOpcodeForwardInnerGuaranteedValues(value) || isa(value)) { foundGuaranteedForwardingPhiOperand = true; diff --git a/lib/SILOptimizer/Analysis/RegionAnalysis.cpp b/lib/SILOptimizer/Analysis/RegionAnalysis.cpp index 8c931e8c83e00..5f895837e6bce 100644 --- a/lib/SILOptimizer/Analysis/RegionAnalysis.cpp +++ b/lib/SILOptimizer/Analysis/RegionAnalysis.cpp @@ -2462,6 +2462,7 @@ CONSTANT_TRANSLATION(TupleInst, Assign) // underlying region. CONSTANT_TRANSLATION(BeginAccessInst, LookThrough) CONSTANT_TRANSLATION(BeginBorrowInst, LookThrough) +CONSTANT_TRANSLATION(BorrowedFromInst, LookThrough) CONSTANT_TRANSLATION(BeginDeallocRefInst, LookThrough) CONSTANT_TRANSLATION(BridgeObjectToRefInst, LookThrough) CONSTANT_TRANSLATION(CopyValueInst, LookThrough) diff --git a/lib/SILOptimizer/LoopTransforms/LoopRotate.cpp b/lib/SILOptimizer/LoopTransforms/LoopRotate.cpp index 88e21b1e51a38..1dbac3b19a07a 100644 --- a/lib/SILOptimizer/LoopTransforms/LoopRotate.cpp +++ b/lib/SILOptimizer/LoopTransforms/LoopRotate.cpp @@ -24,6 +24,7 @@ #include "swift/SILOptimizer/Utils/BasicBlockOptUtils.h" #include "swift/SILOptimizer/Utils/CFGOptUtils.h" #include "swift/SILOptimizer/Utils/LoopUtils.h" +#include "swift/SILOptimizer/Utils/OwnershipOptUtils.h" #include "swift/SILOptimizer/Utils/SILSSAUpdater.h" #include "swift/SILOptimizer/Utils/SILInliner.h" @@ -42,6 +43,10 @@ static llvm::cl::opt LoopRotateSizeLimit("looprotate-size-limit", static llvm::cl::opt RotateSingleBlockLoop("looprotate-single-block-loop", llvm::cl::init(false)); +static bool rotateLoop(SILLoop *loop, DominanceInfo *domInfo, + SILLoopInfo *loopInfo, bool rotateSingleBlockLoops, + SILBasicBlock *upToBB, bool shouldVerify); + /// Check whether all operands are loop invariant. static bool hasLoopInvariantOperands(SILInstruction *inst, SILLoop *loop, @@ -308,7 +313,7 @@ static bool isSingleBlockLoop(SILLoop *L) { /// /// Note: The code relies on the 'UpTo' basic block to stay within the rotate /// loop for termination. -bool swift::rotateLoop(SILLoop *loop, DominanceInfo *domInfo, +static bool rotateLoop(SILLoop *loop, DominanceInfo *domInfo, SILLoopInfo *loopInfo, bool rotateSingleBlockLoops, SILBasicBlock *upToBB, bool shouldVerify) { assert(loop != nullptr && domInfo != nullptr && loopInfo != nullptr @@ -411,7 +416,9 @@ bool swift::rotateLoop(SILLoop *loop, DominanceInfo *domInfo, } for (auto &inst : *header) { - if (SILInstruction *cloned = inst.clone(preheaderBranch)) { + if (auto *bfi = dyn_cast(&inst)) { + valueMap[bfi] = valueMap[bfi->getBorrowedValue()]; + } else if (SILInstruction *cloned = inst.clone(preheaderBranch)) { mapOperands(cloned, valueMap); // The actual operand will sort out which result idx to use. @@ -506,6 +513,7 @@ class LoopRotation : public SILFunctionTransform { } if (changed) { + updateBorrowedFrom(PM, f); // We preserve loop info and the dominator tree. domAnalysis->lockInvalidation(); loopAnalysis->lockInvalidation(); diff --git a/lib/SILOptimizer/Mandatory/ClosureLifetimeFixup.cpp b/lib/SILOptimizer/Mandatory/ClosureLifetimeFixup.cpp index 112873aa5cad5..e4a86cfffa592 100644 --- a/lib/SILOptimizer/Mandatory/ClosureLifetimeFixup.cpp +++ b/lib/SILOptimizer/Mandatory/ClosureLifetimeFixup.cpp @@ -26,6 +26,7 @@ #include "swift/SILOptimizer/PassManager/Transforms.h" #include "swift/SILOptimizer/Utils/BasicBlockOptUtils.h" #include "swift/SILOptimizer/Utils/CFGOptUtils.h" +#include "swift/SILOptimizer/Utils/OwnershipOptUtils.h" #include "swift/SILOptimizer/Utils/InstOptUtils.h" #include "swift/SILOptimizer/Utils/SILSSAUpdater.h" #include "swift/SILOptimizer/Utils/StackNesting.h" @@ -1481,6 +1482,7 @@ class ClosureLifetimeFixup : public SILFunctionTransform { if (fixupClosureLifetimes(*getFunction(), dominanceAnalysis, checkStackNesting, modifiedCFG)) { + updateBorrowedFrom(getPassManager(), getFunction()); if (checkStackNesting){ modifiedCFG |= StackNesting::fixNesting(getFunction()) == StackNesting::Changes::CFG; diff --git a/lib/SILOptimizer/Mandatory/MandatoryInlining.cpp b/lib/SILOptimizer/Mandatory/MandatoryInlining.cpp index f11e5abcf773b..67acaf247a5d1 100644 --- a/lib/SILOptimizer/Mandatory/MandatoryInlining.cpp +++ b/lib/SILOptimizer/Mandatory/MandatoryInlining.cpp @@ -767,9 +767,9 @@ getCalleeFunction(SILFunction *F, FullApplySite AI, bool &IsThick, return CalleeFunction; } -static SILInstruction *tryDevirtualizeApplyHelper(FullApplySite InnerAI, +static SILInstruction *tryDevirtualizeApplyHelper(SILPassManager *pm, FullApplySite InnerAI, ClassHierarchyAnalysis *CHA) { - auto NewInst = tryDevirtualizeApply(InnerAI, CHA).first; + auto NewInst = tryDevirtualizeApply(pm, InnerAI, CHA).first; if (!NewInst) return InnerAI.getInstruction(); @@ -799,7 +799,8 @@ static SILInstruction *tryDevirtualizeApplyHelper(FullApplySite InnerAI, /// /// \returns true if successful, false if failed due to circular inlining. static bool -runOnFunctionRecursively(SILOptFunctionBuilder &FuncBuilder, SILFunction *F, +runOnFunctionRecursively(SILOptFunctionBuilder &FuncBuilder, SILPassManager *pm, + SILFunction *F, FullApplySite AI, DenseFunctionSet &FullyInlinedSet, ImmutableFunctionSet::Factory &SetFactory, ImmutableFunctionSet CurrentInliningSet, @@ -850,7 +851,7 @@ runOnFunctionRecursively(SILOptFunctionBuilder &FuncBuilder, SILFunction *F, // *NOTE* If devirtualization succeeds, devirtInst may not be InnerAI, // but a casted result of InnerAI or even a block argument due to // abstraction changes when calling the witness or class method. - auto *devirtInst = tryDevirtualizeApplyHelper(InnerAI, CHA); + auto *devirtInst = tryDevirtualizeApplyHelper(pm, InnerAI, CHA); // If devirtualization succeeds, make sure we record that this function // changed. if (devirtInst != InnerAI.getInstruction()) @@ -874,7 +875,7 @@ runOnFunctionRecursively(SILOptFunctionBuilder &FuncBuilder, SILFunction *F, // Then recursively process it first before trying to inline it. if (!runOnFunctionRecursively( - FuncBuilder, CalleeFunction, InnerAI, FullyInlinedSet, SetFactory, + FuncBuilder, pm, CalleeFunction, InnerAI, FullyInlinedSet, SetFactory, CurrentInliningSet, CHA, changedFunctions)) { // If we failed due to circular inlining, then emit some notes to // trace back the failure if we have more information. @@ -1035,7 +1036,7 @@ class MandatoryInlining : public SILModuleTransform { if (F.wasDeserializedCanonical()) continue; - runOnFunctionRecursively(FuncBuilder, &F, FullApplySite(), + runOnFunctionRecursively(FuncBuilder, getPassManager(), &F, FullApplySite(), FullyInlinedSet, SetFactory, SetFactory.getEmptySet(), CHA, changedFunctions); diff --git a/lib/SILOptimizer/Mandatory/OwnershipModelEliminator.cpp b/lib/SILOptimizer/Mandatory/OwnershipModelEliminator.cpp index 6fb9e92b19a74..a623062a06641 100644 --- a/lib/SILOptimizer/Mandatory/OwnershipModelEliminator.cpp +++ b/lib/SILOptimizer/Mandatory/OwnershipModelEliminator.cpp @@ -152,6 +152,10 @@ struct OwnershipModelEliminatorVisitor eraseInstructionAndRAUW(bbi, bbi->getOperand()); return true; } + bool visitBorrowedFromInst(BorrowedFromInst *bfi) { + eraseInstructionAndRAUW(bfi, bfi->getBorrowedValue()); + return true; + } bool visitEndBorrowInst(EndBorrowInst *ebi) { eraseInstruction(ebi); return true; diff --git a/lib/SILOptimizer/PassManager/PassManager.cpp b/lib/SILOptimizer/PassManager/PassManager.cpp index ad81bffb1b339..e56c1c333c324 100644 --- a/lib/SILOptimizer/PassManager/PassManager.cpp +++ b/lib/SILOptimizer/PassManager/PassManager.cpp @@ -1618,8 +1618,9 @@ bool BridgedPassContext::tryDeleteDeadClosure(BridgedInstruction closure, bool n BridgedPassContext::DevirtResult BridgedPassContext::tryDevirtualizeApply(BridgedInstruction apply, bool isMandatory) const { - auto cha = invocation->getPassManager()->getAnalysis(); - auto result = ::tryDevirtualizeApply(ApplySite(apply.unbridged()), cha, + SILPassManager *pm = invocation->getPassManager(); + auto cha = pm->getAnalysis(); + auto result = ::tryDevirtualizeApply(pm, ApplySite(apply.unbridged()), cha, nullptr, isMandatory); if (result.first) { OptionalBridgedInstruction newApply(result.first.getInstruction()->asSILNode()); diff --git a/lib/SILOptimizer/SILCombiner/SILCombinerMiscVisitors.cpp b/lib/SILOptimizer/SILCombiner/SILCombinerMiscVisitors.cpp index a2d96e6e8f613..9346561e4e6f7 100644 --- a/lib/SILOptimizer/SILCombiner/SILCombinerMiscVisitors.cpp +++ b/lib/SILOptimizer/SILCombiner/SILCombinerMiscVisitors.cpp @@ -2025,7 +2025,8 @@ SILCombiner::visitDifferentiableFunctionExtractInst(DifferentiableFunctionExtrac return nullptr; std::tie(newValue, std::ignore) = - castValueToABICompatibleType(&Builder, DFEI->getLoc(), + castValueToABICompatibleType(&Builder, parentTransform->getPassManager(), + DFEI->getLoc(), newValue, newValue->getType(), DFEI->getType(), {}); } diff --git a/lib/SILOptimizer/SemanticARC/Context.h b/lib/SILOptimizer/SemanticARC/Context.h index feb8c8754e2f1..b403d9b6149ee 100644 --- a/lib/SILOptimizer/SemanticARC/Context.h +++ b/lib/SILOptimizer/SemanticARC/Context.h @@ -31,6 +31,7 @@ namespace semanticarc { struct LLVM_LIBRARY_VISIBILITY Context { SILFunction &fn; + SILPassManager *pm = nullptr; ARCTransformKind transformKind = ARCTransformKind::All; DeadEndBlocks &deadEndBlocks; ValueLifetimeAnalysis::Frontier lifetimeFrontier; @@ -119,9 +120,9 @@ struct LLVM_LIBRARY_VISIBILITY Context { DeadEndBlocks &getDeadEndBlocks() { return deadEndBlocks; } - Context(SILFunction &fn, DeadEndBlocks &deBlocks, bool onlyMandatoryOpts, + Context(SILFunction &fn, SILPassManager *pm, DeadEndBlocks &deBlocks, bool onlyMandatoryOpts, InstModCallbacks callbacks) - : fn(fn), deadEndBlocks(deBlocks), lifetimeFrontier(), + : fn(fn), pm(pm), deadEndBlocks(deBlocks), lifetimeFrontier(), addressToExhaustiveWriteListCache(constructCacheValue), onlyMandatoryOpts(onlyMandatoryOpts), instModCallbacks(callbacks) {} diff --git a/lib/SILOptimizer/SemanticARC/OwnedToGuaranteedPhiOpt.cpp b/lib/SILOptimizer/SemanticARC/OwnedToGuaranteedPhiOpt.cpp index 4f4ee8e45cbff..8932c97efdac5 100644 --- a/lib/SILOptimizer/SemanticARC/OwnedToGuaranteedPhiOpt.cpp +++ b/lib/SILOptimizer/SemanticARC/OwnedToGuaranteedPhiOpt.cpp @@ -21,6 +21,7 @@ #include "Transforms.h" #include "swift/Basic/Defer.h" #include "swift/Basic/STLExtras.h" +#include "swift/SILOptimizer/Utils/OwnershipOptUtils.h" using namespace swift; using namespace swift::semanticarc; @@ -265,5 +266,8 @@ bool swift::semanticarc::tryConvertOwnedPhisToGuaranteedPhis(Context &ctx) { ctx.verify(); } + if (madeChange) + updateBorrowedFrom(ctx.pm, &ctx.fn); + return madeChange; } diff --git a/lib/SILOptimizer/SemanticARC/OwnershipLiveRange.cpp b/lib/SILOptimizer/SemanticARC/OwnershipLiveRange.cpp index d5e7063c96809..d15e472dffa73 100644 --- a/lib/SILOptimizer/SemanticARC/OwnershipLiveRange.cpp +++ b/lib/SILOptimizer/SemanticARC/OwnershipLiveRange.cpp @@ -105,8 +105,10 @@ OwnershipLiveRange::OwnershipLiveRange(SILValue value) // the users force the live range to be alive. if (!ti) { for (SILValue v : user->getResults()) { - if (v->getOwnershipKind() != OwnershipKind::Owned) + if (v->getOwnershipKind() != OwnershipKind::Owned && + !isa(user)) { continue; + } llvm::copy(v->getUses(), std::back_inserter(worklist)); } continue; diff --git a/lib/SILOptimizer/SemanticARC/SemanticARCOptVisitor.h b/lib/SILOptimizer/SemanticARC/SemanticARCOptVisitor.h index d2405dfe48b8c..f4a59a4709a1f 100644 --- a/lib/SILOptimizer/SemanticARC/SemanticARCOptVisitor.h +++ b/lib/SILOptimizer/SemanticARC/SemanticARCOptVisitor.h @@ -49,9 +49,9 @@ struct LLVM_LIBRARY_VISIBILITY SemanticARCOptVisitor Context ctx; - explicit SemanticARCOptVisitor(SILFunction &fn, DeadEndBlocks &deBlocks, + explicit SemanticARCOptVisitor(SILFunction &fn, SILPassManager *pm, DeadEndBlocks &deBlocks, bool onlyMandatoryOpts) - : ctx(fn, deBlocks, onlyMandatoryOpts, + : ctx(fn, pm, deBlocks, onlyMandatoryOpts, InstModCallbacks() .onDelete( [this](SILInstruction *inst) { eraseInstruction(inst); }) @@ -168,6 +168,7 @@ struct LLVM_LIBRARY_VISIBILITY SemanticARCOptVisitor } \ return false; \ } + FORWARDING_INST(BorrowedFrom) FORWARDING_INST(Tuple) FORWARDING_INST(Object) FORWARDING_INST(Struct) diff --git a/lib/SILOptimizer/SemanticARC/SemanticARCOpts.cpp b/lib/SILOptimizer/SemanticARC/SemanticARCOpts.cpp index 3dbd4712af611..1e814033418a7 100644 --- a/lib/SILOptimizer/SemanticARC/SemanticARCOpts.cpp +++ b/lib/SILOptimizer/SemanticARC/SemanticARCOpts.cpp @@ -157,7 +157,7 @@ struct SemanticARCOpts : SILFunctionTransform { "verification is enabled"); auto *deBlocksAnalysis = getAnalysis(); - SemanticARCOptVisitor visitor(f, *deBlocksAnalysis->get(&f), + SemanticARCOptVisitor visitor(f, getPassManager(), *deBlocksAnalysis->get(&f), mandatoryOptsOnly); #ifndef NDEBUG diff --git a/lib/SILOptimizer/Transforms/ConditionForwarding.cpp b/lib/SILOptimizer/Transforms/ConditionForwarding.cpp index 07db9926dffab..42e30b1d788d8 100644 --- a/lib/SILOptimizer/Transforms/ConditionForwarding.cpp +++ b/lib/SILOptimizer/Transforms/ConditionForwarding.cpp @@ -18,6 +18,7 @@ #include "swift/SIL/SILBuilder.h" #include "swift/SIL/SILInstruction.h" #include "swift/SIL/SILUndef.h" +#include "swift/SILOptimizer/Utils/OwnershipOptUtils.h" #include "swift/SILOptimizer/PassManager/Passes.h" #include "swift/SILOptimizer/PassManager/Transforms.h" @@ -115,6 +116,7 @@ class ConditionForwarding : public SILFunctionTransform { } } if (Changed) { + updateBorrowedFrom(getPassManager(), F); invalidateAnalysis(SILAnalysis::InvalidationKind::BranchesAndInstructions); } } @@ -158,13 +160,15 @@ static bool hasNoRelevantSideEffects(SILBasicBlock *BB) { bool ConditionForwarding::tryOptimize(SwitchEnumInst *SEI) { // The switch_enum argument (an Enum) must be a block argument at the merging // point of the condition's destinations. - auto *Arg = dyn_cast(SEI->getOperand()); + auto *Arg = dyn_cast(lookThroughBorrowedFromDef(SEI->getOperand())); if (!Arg) return false; + SILValue argValue = lookThroughBorrowedFromUser(Arg); + // The switch_enum must be the only use of the Enum, except it may be used in // SEI's successors. - for (Operand *ArgUse : Arg->getUses()) { + for (Operand *ArgUse : argValue->getUses()) { SILInstruction *ArgUser = ArgUse->getUser(); if (ArgUser == SEI) continue; @@ -242,8 +246,8 @@ bool ConditionForwarding::tryOptimize(SwitchEnumInst *SEI) { // First thing to do is to replace all uses of the Enum (= the merging block // argument), as this argument gets deleted. BasicBlockSet NeedEnumArg(BB->getParent()); - while (!Arg->use_empty()) { - Operand *ArgUse = *Arg->use_begin(); + while (!argValue->use_empty()) { + Operand *ArgUse = *argValue->use_begin(); SILInstruction *ArgUser = ArgUse->getUser(); if (ArgUser->isDebugInstruction()) { // Don't care about debug instructions. Just remove them. @@ -317,6 +321,9 @@ bool ConditionForwarding::tryOptimize(SwitchEnumInst *SEI) { } } } + if (argValue != Arg) { + cast(argValue)->eraseFromParent(); + } // Final step: replace the switch_enum by the condition. SILBuilder B(Condition); diff --git a/lib/SILOptimizer/Transforms/DeadCodeElimination.cpp b/lib/SILOptimizer/Transforms/DeadCodeElimination.cpp index 70ec36d2ec54d..55be398ba9a7c 100644 --- a/lib/SILOptimizer/Transforms/DeadCodeElimination.cpp +++ b/lib/SILOptimizer/Transforms/DeadCodeElimination.cpp @@ -285,7 +285,7 @@ void DCE::markLive() { break; } case SILInstructionKind::EndBorrowInst: { - auto phi = PhiValue(I.getOperand(0)); + auto phi = PhiValue(lookThroughBorrowedFromDef(I.getOperand(0))); // If there is a pointer escape or phi is lexical, disable DCE. if (phi && (findPointerEscape(phi) || phi->isLexical())) { markInstructionLive(&I); @@ -294,6 +294,10 @@ void DCE::markLive() { addReverseDependency(I.getOperand(0), &I); break; } + case SILInstructionKind::BorrowedFromInst: { + addReverseDependency(I.getOperand(0), &I); + break; + } case SILInstructionKind::EndLifetimeInst: { // The instruction is live only if it's operand value is also live addReverseDependency(I.getOperand(0), &I); @@ -347,7 +351,7 @@ void DCE::markLive() { // Records a reverse dependency if needed. See DCE::ReverseDependencies. void DCE::addReverseDependency(SILValue from, SILInstruction *to) { LLVM_DEBUG(llvm::dbgs() << "Adding reverse dependency from " << from << " to " - << to); + << *to); ReverseDependencies[from].insert(to); } @@ -576,7 +580,7 @@ void DCE::endLifetimeOfLiveValue(SILValue value, SILInstruction *insertPt) { builder.emitDestroyOperation(RegularLocation::getAutoGeneratedLocation(), value); } - BorrowedValue borrow(value); + BorrowedValue borrow(lookThroughBorrowedFromDef(value)); if (borrow && borrow.isLocalScope()) { builder.emitEndBorrowOperation(RegularLocation::getAutoGeneratedLocation(), value); diff --git a/lib/SILOptimizer/Transforms/Devirtualizer.cpp b/lib/SILOptimizer/Transforms/Devirtualizer.cpp index 25cdfffd73fc9..1a62cabed32e9 100644 --- a/lib/SILOptimizer/Transforms/Devirtualizer.cpp +++ b/lib/SILOptimizer/Transforms/Devirtualizer.cpp @@ -78,7 +78,7 @@ void Devirtualizer::devirtualizeAppliesInFunction(SILFunction &F, for (auto Apply : Applies) { ApplySite NewInst; bool modifiedCFG; - std::tie(NewInst, modifiedCFG) = tryDevirtualizeApply(Apply, CHA, &ORE); + std::tie(NewInst, modifiedCFG) = tryDevirtualizeApply(getPassManager(), Apply, CHA, &ORE); if (!NewInst) continue; diff --git a/lib/SILOptimizer/Transforms/PerformanceInliner.cpp b/lib/SILOptimizer/Transforms/PerformanceInliner.cpp index 1568cf62ba552..6a8faebf939de 100644 --- a/lib/SILOptimizer/Transforms/PerformanceInliner.cpp +++ b/lib/SILOptimizer/Transforms/PerformanceInliner.cpp @@ -21,6 +21,7 @@ #include "swift/SILOptimizer/Utils/CFGOptUtils.h" #include "swift/SILOptimizer/Utils/Devirtualize.h" #include "swift/SILOptimizer/Utils/Generics.h" +#include "swift/SILOptimizer/Utils/OwnershipOptUtils.h" #include "swift/SILOptimizer/Utils/PerformanceInlinerUtils.h" #include "swift/SILOptimizer/Utils/SILOptFunctionBuilder.h" #include "swift/SILOptimizer/Utils/StackNesting.h" @@ -1207,6 +1208,7 @@ bool SILPerformanceInliner::inlineCallsIntoFunction(SILFunction *Caller) { if (invalidatedStackNesting) { StackNesting::fixNesting(Caller); } + updateBorrowedFrom(pm, Caller); // If we were asked to verify our caller after inlining all callees we could // find into it, do so now. This makes it easier to catch verification bugs in diff --git a/lib/SILOptimizer/Transforms/SimplifyCFG.cpp b/lib/SILOptimizer/Transforms/SimplifyCFG.cpp index c730958c05f5f..901bcf9bbf7a7 100644 --- a/lib/SILOptimizer/Transforms/SimplifyCFG.cpp +++ b/lib/SILOptimizer/Transforms/SimplifyCFG.cpp @@ -180,7 +180,7 @@ bool SimplifyCFG::threadEdge(const ThreadInfo &ti) { << " to bb" << ti.Dest->getDebugID() << '\n'); auto *SrcTerm = cast(ti.Src->getTerminator()); - BasicBlockCloner Cloner(SrcTerm->getDestBB()); + BasicBlockCloner Cloner(SrcTerm->getDestBB(), PM); if (!Cloner.canCloneBlock()) return false; @@ -571,7 +571,7 @@ bool SimplifyCFG::dominatorBasedSimplify(DominanceAnalysis *DA) { // Do dominator based simplification of terminator condition. This does not // and MUST NOT change the CFG without updating the dominator tree to // reflect such change. - if (tryCheckedCastBrJumpThreading(&Fn, DT, deBlocks, BlocksForWorklist, + if (tryCheckedCastBrJumpThreading(&Fn, PM, DT, deBlocks, BlocksForWorklist, EnableOSSACheckedCastBrJumpThreading)) { for (auto BB: BlocksForWorklist) addToWorklist(BB); @@ -1023,7 +1023,7 @@ bool SimplifyCFG::tryJumpThreading(BranchInst *BI) { // If it looks potentially interesting, decide whether we *can* do the // operation and whether the block is small enough to be worth duplicating. int copyCosts = 0; - BasicBlockCloner Cloner(DestBB); + BasicBlockCloner Cloner(DestBB, PM); for (auto &inst : *DestBB) { copyCosts += getThreadingCost(&inst); if (ThreadingBudget <= copyCosts) @@ -1287,7 +1287,13 @@ bool SimplifyCFG::simplifyBranchBlock(BranchInst *BI) { for (unsigned i = 0, e = BI->getArgs().size(); i != e; ++i) { assert(DestBB->getArgument(i) != BI->getArg(i)); SILValue Val = BI->getArg(i); - DestBB->getArgument(i)->replaceAllUsesWith(Val); + SILValue arg = DestBB->getArgument(i); + if (auto *bfi = getBorrowedFromUser(arg)) { + bfi->replaceAllUsesWith(Val); + bfi->eraseFromParent(); + } else { + arg->replaceAllUsesWith(Val); + } if (!isVeryLargeFunction) { if (auto *I = dyn_cast(Val)) { // Replacing operands may trigger constant folding which then could @@ -1545,8 +1551,12 @@ bool SimplifyCFG::simplifyCondBrBlock(CondBranchInst *BI) { // Erase in reverse order to pop each element as we go. for (unsigned i = destBB->getArguments().size(); i != 0;) { --i; - destBB->getArgument(i)->replaceAllUsesWith( - trampolineDest.newSourceBranchArgs[i]); + SILArgument *arg = destBB->getArgument(i); + if (auto *bfi = getBorrowedFromUser(arg)) { + bfi->replaceAllUsesWith(arg); + bfi->eraseFromParent(); + } + arg->replaceAllUsesWith(trampolineDest.newSourceBranchArgs[i]); destBB->eraseArgument(i); } }; @@ -2176,11 +2186,13 @@ bool SimplifyCFG::simplifySwitchEnumBlock(SwitchEnumInst *SEI) { EnumCase.get()); } Builder.createBranch(loc, LiveBlock, PayLoad); + SEI->eraseFromParent(); + updateBorrowedFromPhis(PM, { cast(LiveBlock->getArgument(0)) }); } else { Builder.createBranch(loc, LiveBlock); + SEI->eraseFromParent(); } - SEI->eraseFromParent(); if (EI && isInstructionTriviallyDead(EI)) { EI->replaceAllUsesOfAllResultsWithUndef(); EI->eraseFromParent(); @@ -2561,7 +2573,7 @@ bool SimplifyCFG::simplifyTryApplyBlock(TryApplyInst *TAI) { } // Cast argument if required. std::tie(Arg, std::ignore) = castValueToABICompatibleType( - &Builder, TAI->getLoc(), Arg, origConv.getSILArgumentType(i, context), + &Builder, PM, TAI->getLoc(), Arg, origConv.getSILArgumentType(i, context), targetConv.getSILArgumentType(calleeArgIdx, context), {TAI}); Args.push_back(Arg); calleeArgIdx += 1; @@ -2596,7 +2608,7 @@ bool SimplifyCFG::simplifyTryApplyBlock(TryApplyInst *TAI) { // Non-guaranteed values don't need use points when casting. SILValue CastedResult; std::tie(CastedResult, std::ignore) = castValueToABICompatibleType( - &Builder, Loc, NewAI, ResultTy, OrigResultTy, /*usePoints*/ {}); + &Builder, PM, Loc, NewAI, ResultTy, OrigResultTy, /*usePoints*/ {}); BranchInst *branch = Builder.createBranch(Loc, NormalBB, { CastedResult }); @@ -2985,7 +2997,7 @@ bool SimplifyCFG::tailDuplicateObjCMethodCallSuccessorBlocks() { // Okay, it looks like we want to do this and we can. Duplicate the // destination block into this one, rewriting uses of the BBArgs to use the // branch arguments as we go. - BasicBlockCloner Cloner(DestBB); + BasicBlockCloner Cloner(DestBB, PM); if (!Cloner.canCloneBlock()) continue; @@ -3691,10 +3703,11 @@ bool SimplifyCFG::simplifyArgument(SILBasicBlock *BB, unsigned i) { return simplifySwitchEnumToSelectEnum(BB, i, A); // For now, just focus on cases where there is a single use. - if (!A->hasOneUse()) + SILValue argVal = lookThroughBorrowedFromUser(A); + if (!argVal->hasOneUse()) return false; - auto *Use = *A->use_begin(); + auto *Use = *argVal->use_begin(); auto *User = Use->getUser(); auto disableInOSSA = [](SingleValueInstruction *inst) { @@ -3740,6 +3753,10 @@ bool SimplifyCFG::simplifyArgument(SILBasicBlock *BB, unsigned i) { // Okay, we'll replace the BB arg with one with the right type, replace // the uses in this block, and then rewrite the branch operands. LLVM_DEBUG(llvm::dbgs() << "unwrap argument:" << *A); + if (auto *bfi = getBorrowedFromUser(A)) { + bfi->replaceAllUsesWith(A); + bfi->eraseFromParent(); + } A->replaceAllUsesWith(SILUndef::get(A)); auto *NewArg = BB->replacePhiArgument(i, proj->getType(), BB->getArgument(i)->getOwnershipKind()); @@ -3762,6 +3779,8 @@ bool SimplifyCFG::simplifyArgument(SILBasicBlock *BB, unsigned i) { proj->eraseFromParent(); + updateBorrowedFromPhis(PM, { NewArg }); + return true; } @@ -3821,6 +3840,10 @@ static void tryToReplaceArgWithIncomingValue(SILBasicBlock *BB, unsigned i, // An argument has one result value. We need to replace this with the *value* // of the incoming block(s). LLVM_DEBUG(llvm::dbgs() << "replace arg with incoming value:" << *A); + if (auto *bfi = getBorrowedFromUser(A)) { + bfi->replaceAllUsesWith(A); + bfi->eraseFromParent(); + } A->replaceAllUsesWith(V); } diff --git a/lib/SILOptimizer/Transforms/SpeculativeDevirtualizer.cpp b/lib/SILOptimizer/Transforms/SpeculativeDevirtualizer.cpp index 10dbcc72165ad..1db43d0f98779 100644 --- a/lib/SILOptimizer/Transforms/SpeculativeDevirtualizer.cpp +++ b/lib/SILOptimizer/Transforms/SpeculativeDevirtualizer.cpp @@ -122,7 +122,7 @@ static FullApplySite CloneApply(FullApplySite AI, SILValue SelfArg, /// Insert monomorphic inline caches for a specific class or metatype /// type \p SubClassTy. -static FullApplySite speculateMonomorphicTarget(FullApplySite AI, +static FullApplySite speculateMonomorphicTarget(SILPassManager *pm, FullApplySite AI, CanType SubType, ClassDecl *CD, CanType ClassType, CheckedCastBranchInst *&CCBI) { @@ -228,7 +228,7 @@ static FullApplySite speculateMonomorphicTarget(FullApplySite AI, ++NumTargetsPredicted; // Devirtualize the apply instruction on the identical path. - auto NewInst = devirtualizeClassMethod(IdenAI, DownCastedClassInstance, CD, + auto NewInst = devirtualizeClassMethod(pm, IdenAI, DownCastedClassInstance, CD, ClassType, nullptr) .first; assert(NewInst && "Expected to be able to devirtualize apply!"); @@ -378,7 +378,7 @@ static bool isDefaultCaseKnown(ClassHierarchyAnalysis *CHA, /// Try to speculate the call target for the call \p AI. This function /// returns true if a change was made. -static bool tryToSpeculateTarget(FullApplySite AI, ClassHierarchyAnalysis *CHA, +static bool tryToSpeculateTarget(SILPassManager *pm, FullApplySite AI, ClassHierarchyAnalysis *CHA, OptRemark::Emitter &ORE) { ClassMethodInst *CMI = cast(AI.getCallee()); @@ -409,7 +409,7 @@ static bool tryToSpeculateTarget(FullApplySite AI, ClassHierarchyAnalysis *CHA, ClassHierarchyAnalysis::ClassList Subs; if (isDefaultCaseKnown(CHA, AI, CD, Subs)) { auto NewInst = - tryDevirtualizeClassMethod(AI, SubTypeValue, CD, ClassType, &ORE) + tryDevirtualizeClassMethod(pm, AI, SubTypeValue, CD, ClassType, &ORE) .first; if (NewInst) deleteDevirtualizedApply(AI); @@ -418,7 +418,7 @@ static bool tryToSpeculateTarget(FullApplySite AI, ClassHierarchyAnalysis *CHA, LLVM_DEBUG(llvm::dbgs() << "Inserting monomorphic speculative call for " "class " << CD->getName() << "\n"); - return !!speculateMonomorphicTarget(AI, SubType, CD, ClassType, LastCCBI); + return !!speculateMonomorphicTarget(pm, AI, SubType, CD, ClassType, LastCCBI); } // True if any instructions were changed or generated. @@ -454,7 +454,7 @@ static bool tryToSpeculateTarget(FullApplySite AI, ClassHierarchyAnalysis *CHA, } auto FirstAI = - speculateMonomorphicTarget(AI, SubType, CD, ClassType, LastCCBI); + speculateMonomorphicTarget(pm, AI, SubType, CD, ClassType, LastCCBI); if (FirstAI) { Changed = true; AI = FirstAI; @@ -516,7 +516,7 @@ static bool tryToSpeculateTarget(FullApplySite AI, ClassHierarchyAnalysis *CHA, } // Pass the metatype of the subclass. - auto NewAI = speculateMonomorphicTarget(AI, ClassOrMetatypeType, S, + auto NewAI = speculateMonomorphicTarget(pm, AI, ClassOrMetatypeType, S, CanClassType, LastCCBI); if (!NewAI) { ++NotHandledSubsNum; @@ -573,7 +573,7 @@ static bool tryToSpeculateTarget(FullApplySite AI, ClassHierarchyAnalysis *CHA, return true; } auto NewInst = - tryDevirtualizeClassMethod(AI, SubTypeValue, CD, ClassType, nullptr) + tryDevirtualizeClassMethod(pm, AI, SubTypeValue, CD, ClassType, nullptr) .first; if (NewInst) { ORE.emit(RB); @@ -622,7 +622,7 @@ namespace { OptRemark::Emitter ORE(DEBUG_TYPE, CurFn); // Go over the collected calls and try to insert speculative calls. for (auto AI : ToSpecialize) - Changed |= tryToSpeculateTarget(AI, CHA, ORE); + Changed |= tryToSpeculateTarget(getPassManager(), AI, CHA, ORE); if (Changed) { CurFn.getModule().linkFunction(&CurFn, SILModule::LinkingMode::LinkAll); diff --git a/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp b/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp index 3c1ba628ad2dc..ef2ac67ffa996 100644 --- a/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp +++ b/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp @@ -222,6 +222,7 @@ static bool hasOpaqueArchetype(TypeExpansionContext context, case SILInstructionKind::LoadInst: case SILInstructionKind::LoadBorrowInst: case SILInstructionKind::BeginBorrowInst: + case SILInstructionKind::BorrowedFromInst: case SILInstructionKind::StoreBorrowInst: case SILInstructionKind::BeginAccessInst: case SILInstructionKind::MoveOnlyWrapperToCopyableAddrInst: diff --git a/lib/SILOptimizer/Utils/BasicBlockOptUtils.cpp b/lib/SILOptimizer/Utils/BasicBlockOptUtils.cpp index 05895b018fb4b..56e6566fcd66e 100644 --- a/lib/SILOptimizer/Utils/BasicBlockOptUtils.cpp +++ b/lib/SILOptimizer/Utils/BasicBlockOptUtils.cpp @@ -193,6 +193,7 @@ void BasicBlockCloner::updateSSAAfterCloning() { ssaUpdater.rewriteUse(*use); } } + updateBorrowedFromPhis(pm, updateSSAPhis); } void BasicBlockCloner::sinkAddressProjections() { diff --git a/lib/SILOptimizer/Utils/CFGOptUtils.cpp b/lib/SILOptimizer/Utils/CFGOptUtils.cpp index 9c6e74b5cf852..4d7c7811b548b 100644 --- a/lib/SILOptimizer/Utils/CFGOptUtils.cpp +++ b/lib/SILOptimizer/Utils/CFGOptUtils.cpp @@ -19,6 +19,7 @@ #include "swift/SIL/SILArgument.h" #include "swift/SIL/SILBuilder.h" #include "swift/SIL/BasicBlockDatastructures.h" +#include "swift/SIL/OwnershipUtils.h" #include "swift/SILOptimizer/Utils/InstOptUtils.h" #include "llvm/ADT/TinyPtrVector.h" @@ -142,8 +143,12 @@ TermInst *swift::deleteEdgeValue(TermInst *branch, SILBasicBlock *destBlock, void swift::erasePhiArgument(SILBasicBlock *block, unsigned argIndex, bool cleanupDeadPhiOps, InstModCallbacks callbacks) { - assert(block->getArgument(argIndex)->isPhi() - && "Only should be used on phi arguments"); + SILArgument *arg = block->getArgument(argIndex); + assert(arg->isPhi() && "Only should be used on phi arguments"); + if (auto *bfi = getBorrowedFromUser(arg)) { + bfi->replaceAllUsesWith(arg); + bfi->eraseFromParent(); + } block->eraseArgument(argIndex); // Determine the set of predecessors in case any predecessor has diff --git a/lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp b/lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp index 249b56b6051ea..2a2795a7b0462 100644 --- a/lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp +++ b/lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp @@ -155,6 +155,10 @@ bool CanonicalizeOSSALifetime::computeCanonicalLiveness() { defUseWorklist.insert(copy); continue; } + if (auto *bfi = dyn_cast(user)) { + defUseWorklist.insert(bfi); + continue; + } // Handle debug_value instructions separately. if (pruneDebugMode) { if (auto *dvi = dyn_cast(user)) { diff --git a/lib/SILOptimizer/Utils/CheckedCastBrJumpThreading.cpp b/lib/SILOptimizer/Utils/CheckedCastBrJumpThreading.cpp index 0b73b50d7f0a0..6090f0a51ff53 100644 --- a/lib/SILOptimizer/Utils/CheckedCastBrJumpThreading.cpp +++ b/lib/SILOptimizer/Utils/CheckedCastBrJumpThreading.cpp @@ -50,6 +50,8 @@ class CheckedCastBrJumpThreading { // need to be recomputed each time tryCheckedCastBrJumpThreading is called. DeadEndBlocks *deBlocks; + SILPassManager *pm; + // Enable non-trivial terminator rewriting in OSSA. bool EnableOSSARewriteTerminator; @@ -141,10 +143,10 @@ class CheckedCastBrJumpThreading { public: CheckedCastBrJumpThreading( - SILFunction *Fn, DominanceInfo *DT, DeadEndBlocks *deBlocks, + SILFunction *Fn, SILPassManager *pm, DominanceInfo *DT, DeadEndBlocks *deBlocks, SmallVectorImpl &BlocksForWorklist, bool EnableOSSARewriteTerminator) - : Fn(Fn), DT(DT), deBlocks(deBlocks), + : Fn(Fn), DT(DT), deBlocks(deBlocks), pm(pm), EnableOSSARewriteTerminator(EnableOSSARewriteTerminator), rauwContext(callbacks, *deBlocks), BlocksForWorklist(BlocksForWorklist), BlocksToEdit(Fn), @@ -772,7 +774,7 @@ void CheckedCastBrJumpThreading::optimizeFunction() { if (edit->SuccessArg->isErased()) continue; - BasicBlockCloner Cloner(edit->CCBBlock, deBlocks); + BasicBlockCloner Cloner(edit->CCBBlock, pm, deBlocks); if (!Cloner.canCloneBlock()) continue; @@ -800,7 +802,7 @@ void CheckedCastBrJumpThreading::optimizeFunction() { namespace swift { bool tryCheckedCastBrJumpThreading( - SILFunction *Fn, DominanceInfo *DT, DeadEndBlocks *deBlocks, + SILFunction *Fn, SILPassManager *pm, DominanceInfo *DT, DeadEndBlocks *deBlocks, SmallVectorImpl &BlocksForWorklist, bool EnableOSSARewriteTerminator) { @@ -809,7 +811,7 @@ bool tryCheckedCastBrJumpThreading( return false; } - CheckedCastBrJumpThreading CCBJumpThreading(Fn, DT, deBlocks, + CheckedCastBrJumpThreading CCBJumpThreading(Fn, pm, DT, deBlocks, BlocksForWorklist, EnableOSSARewriteTerminator); CCBJumpThreading.optimizeFunction(); diff --git a/lib/SILOptimizer/Utils/Devirtualize.cpp b/lib/SILOptimizer/Utils/Devirtualize.cpp index 55731d538510b..1fedb7aa1070a 100644 --- a/lib/SILOptimizer/Utils/Devirtualize.cpp +++ b/lib/SILOptimizer/Utils/Devirtualize.cpp @@ -452,7 +452,7 @@ getSubstitutionsForCallee(SILModule &module, CanSILFunctionType baseCalleeType, // Return the new apply and true if a cast required CFG modification. static std::pair -replaceApplyInst(SILBuilder &builder, SILLocation loc, ApplyInst *oldAI, +replaceApplyInst(SILBuilder &builder, SILPassManager *pm, SILLocation loc, ApplyInst *oldAI, SILValue newFn, SubstitutionMap newSubs, ArrayRef newArgs, ArrayRef newArgBorrows) { auto *newAI = @@ -469,7 +469,7 @@ replaceApplyInst(SILBuilder &builder, SILLocation loc, ApplyInst *oldAI, // guaranteed value, so this cast cannot generate borrow scopes and it can be // used anywhere the original oldAI was used. auto castRes = castValueToABICompatibleType( - &builder, loc, newAI, newAI->getType(), oldAI->getType(), /*usePoints*/ {}); + &builder, pm, loc, newAI, newAI->getType(), oldAI->getType(), /*usePoints*/ {}); oldAI->replaceAllUsesWith(castRes.first); return {newAI, castRes.second}; @@ -477,7 +477,7 @@ replaceApplyInst(SILBuilder &builder, SILLocation loc, ApplyInst *oldAI, // Return the new try_apply and true if a cast required CFG modification. static std::pair -replaceTryApplyInst(SILBuilder &builder, SILLocation loc, TryApplyInst *oldTAI, +replaceTryApplyInst(SILBuilder &builder, SILPassManager *pm, SILLocation loc, TryApplyInst *oldTAI, SILValue newFn, SubstitutionMap newSubs, ArrayRef newArgs, SILFunctionConventions conv, ArrayRef newArgBorrows) { @@ -530,7 +530,7 @@ replaceTryApplyInst(SILBuilder &builder, SILLocation loc, TryApplyInst *oldTAI, // borrow scopes and it can be used anywhere the original oldAI was // used--usePoints are not required. std::tie(resultValue, std::ignore) = castValueToABICompatibleType( - &builder, loc, resultValue, newResultTy, oldResultTy, /*usePoints*/ {}); + &builder, pm, loc, resultValue, newResultTy, oldResultTy, /*usePoints*/ {}); builder.createBranch(loc, normalBB, {resultValue}); } @@ -540,7 +540,7 @@ replaceTryApplyInst(SILBuilder &builder, SILLocation loc, TryApplyInst *oldTAI, // Return the new begin_apply and true if a cast required CFG modification. static std::pair -replaceBeginApplyInst(SILBuilder &builder, SILLocation loc, +replaceBeginApplyInst(SILBuilder &builder, SILPassManager *pm, SILLocation loc, BeginApplyInst *oldBAI, SILValue newFn, SubstitutionMap newSubs, ArrayRef newArgs, ArrayRef newArgBorrows) { @@ -562,7 +562,7 @@ replaceBeginApplyInst(SILBuilder &builder, SILLocation loc, SmallVector users( makeUserIteratorRange(oldYield->getUses())); auto yieldCastRes = castValueToABICompatibleType( - &builder, loc, newYield, newYield->getType(), oldYield->getType(), + &builder, pm, loc, newYield, newYield->getType(), oldYield->getType(), users); oldYield->replaceAllUsesWith(yieldCastRes.first); changedCFG |= yieldCastRes.second; @@ -589,7 +589,7 @@ replaceBeginApplyInst(SILBuilder &builder, SILLocation loc, // Return the new partial_apply and true if a cast required CFG modification. static std::pair -replacePartialApplyInst(SILBuilder &builder, SILLocation loc, +replacePartialApplyInst(SILBuilder &builder, SILPassManager *pm, SILLocation loc, PartialApplyInst *oldPAI, SILValue newFn, SubstitutionMap newSubs, ArrayRef newArgs) { auto convention = oldPAI->getCalleeConvention(); @@ -602,7 +602,7 @@ replacePartialApplyInst(SILBuilder &builder, SILLocation loc, // A non-guaranteed cast needs no usePoints. assert(newPAI->getOwnershipKind() != OwnershipKind::Guaranteed); auto castRes = castValueToABICompatibleType( - &builder, loc, newPAI, newPAI->getType(), oldPAI->getType(), + &builder, pm, loc, newPAI, newPAI->getType(), oldPAI->getType(), /*usePoints*/ {}); oldPAI->replaceAllUsesWith(castRes.first); @@ -611,30 +611,30 @@ replacePartialApplyInst(SILBuilder &builder, SILLocation loc, // Return the new apply and true if the CFG was also modified. static std::pair -replaceApplySite(SILBuilder &builder, SILLocation loc, ApplySite oldAS, +replaceApplySite(SILBuilder &builder, SILPassManager *pm, SILLocation loc, ApplySite oldAS, SILValue newFn, SubstitutionMap newSubs, ArrayRef newArgs, SILFunctionConventions conv, ArrayRef newArgBorrows) { switch (oldAS.getKind()) { case ApplySiteKind::ApplyInst: { auto *oldAI = cast(oldAS); - return replaceApplyInst(builder, loc, oldAI, newFn, newSubs, newArgs, + return replaceApplyInst(builder, pm, loc, oldAI, newFn, newSubs, newArgs, newArgBorrows); } case ApplySiteKind::TryApplyInst: { auto *oldTAI = cast(oldAS); - return replaceTryApplyInst(builder, loc, oldTAI, newFn, newSubs, newArgs, + return replaceTryApplyInst(builder, pm, loc, oldTAI, newFn, newSubs, newArgs, conv, newArgBorrows); } case ApplySiteKind::BeginApplyInst: { auto *oldBAI = dyn_cast(oldAS); - return replaceBeginApplyInst(builder, loc, oldBAI, newFn, newSubs, newArgs, + return replaceBeginApplyInst(builder, pm, loc, oldBAI, newFn, newSubs, newArgs, newArgBorrows); } case ApplySiteKind::PartialApplyInst: { assert(newArgBorrows.empty()); auto *oldPAI = cast(oldAS); - return replacePartialApplyInst(builder, loc, oldPAI, newFn, newSubs, + return replacePartialApplyInst(builder, pm, loc, oldPAI, newFn, newSubs, newArgs); } } @@ -749,7 +749,7 @@ bool swift::canDevirtualizeClassMethod(FullApplySite applySite, ClassDecl *cd, /// /// Return the new apply and true if the CFG was also modified. std::pair -swift::devirtualizeClassMethod(FullApplySite applySite, +swift::devirtualizeClassMethod(SILPassManager *pm, FullApplySite applySite, SILValue classOrMetatype, ClassDecl *cd, CanType classType, OptRemark::Emitter *ore) { bool changedCFG = false; @@ -792,7 +792,7 @@ swift::devirtualizeClassMethod(FullApplySite applySite, for (auto resultTy : substConv.getIndirectSILResultTypes( applySite.getFunction()->getTypeExpansionContext())) { auto castRes = castValueToABICompatibleType( - &builder, loc, *indirectResultArgIter, indirectResultArgIter->getType(), + &builder, pm, loc, *indirectResultArgIter, indirectResultArgIter->getType(), resultTy, {applySite.getInstruction()}); newArgs.push_back(castRes.first); changedCFG |= castRes.second; @@ -813,7 +813,7 @@ swift::devirtualizeClassMethod(FullApplySite applySite, newArgBorrows.push_back(arg); } auto argCastRes = - castValueToABICompatibleType(&builder, loc, arg, + castValueToABICompatibleType(&builder, pm, loc, arg, paramArgIter->getType(), paramType, {applySite.getInstruction()}); @@ -824,7 +824,7 @@ swift::devirtualizeClassMethod(FullApplySite applySite, ApplySite newAS; bool neededCFGChange; std::tie(newAS, neededCFGChange) = replaceApplySite( - builder, loc, applySite, fri, subs, newArgs, substConv, newArgBorrows); + builder, pm, loc, applySite, fri, subs, newArgs, substConv, newArgBorrows); changedCFG |= neededCFGChange; FullApplySite newAI = FullApplySite::isa(newAS.getInstruction()); assert(newAI); @@ -843,12 +843,12 @@ swift::devirtualizeClassMethod(FullApplySite applySite, } std::pair swift::tryDevirtualizeClassMethod( - FullApplySite applySite, SILValue classInstance, ClassDecl *cd, + SILPassManager *pm, FullApplySite applySite, SILValue classInstance, ClassDecl *cd, CanType classType, OptRemark::Emitter *ore, bool isEffectivelyFinalMethod) { if (!canDevirtualizeClassMethod(applySite, cd, classType, ore, isEffectivelyFinalMethod)) return {FullApplySite(), false}; - return devirtualizeClassMethod(applySite, classInstance, cd, classType, ore); + return devirtualizeClassMethod(pm, applySite, classInstance, cd, classType, ore); } //===----------------------------------------------------------------------===// @@ -1051,7 +1051,7 @@ swift::getWitnessMethodSubstitutions(SILModule &module, ApplySite applySite, /// /// Return the new apply and true if the CFG was also modified. static std::pair -devirtualizeWitnessMethod(ApplySite applySite, SILFunction *f, +devirtualizeWitnessMethod(SILPassManager *pm, ApplySite applySite, SILFunction *f, ProtocolConformanceRef cRef, OptRemark::Emitter *ore) { bool changedCFG = false; @@ -1099,7 +1099,7 @@ devirtualizeWitnessMethod(ApplySite applySite, SILFunction *f, borrowedArgs.push_back(arg); } auto argCastRes = castValueToABICompatibleType( - &argBuilder, applySite.getLoc(), arg, arg->getType(), paramType, + &argBuilder, pm, applySite.getLoc(), arg, arg->getType(), paramType, applySite.getInstruction()); arg = argCastRes.first; changedCFG |= argCastRes.second; @@ -1117,7 +1117,7 @@ devirtualizeWitnessMethod(ApplySite applySite, SILFunction *f, ApplySite newApplySite; bool neededCFGChange = false; std::tie(newApplySite, neededCFGChange) = - replaceApplySite(applyBuilder, loc, applySite, fri, subMap, arguments, + replaceApplySite(applyBuilder, pm, loc, applySite, fri, subMap, arguments, substConv, borrowedArgs); changedCFG |= neededCFGChange; @@ -1266,7 +1266,7 @@ static bool canDevirtualizeWitnessMethod(ApplySite applySite, bool isMandatory) /// we'll call to, replace an apply of a witness_method with an apply /// of a function_ref, returning the new apply. std::pair -swift::tryDevirtualizeWitnessMethod(ApplySite applySite, +swift::tryDevirtualizeWitnessMethod(SILPassManager *pm, ApplySite applySite, OptRemark::Emitter *ore, bool isMandatory) { if (!canDevirtualizeWitnessMethod(applySite, isMandatory)) @@ -1280,7 +1280,7 @@ swift::tryDevirtualizeWitnessMethod(ApplySite applySite, std::tie(f, wt) = applySite.getModule().lookUpFunctionInWitnessTable( wmi->getConformance(), wmi->getMember(), SILModule::LinkingMode::LinkAll); - return devirtualizeWitnessMethod(applySite, f, wmi->getConformance(), ore); + return devirtualizeWitnessMethod(pm, applySite, f, wmi->getConformance(), ore); } //===----------------------------------------------------------------------===// @@ -1292,7 +1292,7 @@ swift::tryDevirtualizeWitnessMethod(ApplySite applySite, /// /// Return the new apply and true if the CFG was also modified. std::pair -swift::tryDevirtualizeApply(ApplySite applySite, ClassHierarchyAnalysis *cha, +swift::tryDevirtualizeApply(SILPassManager *pm, ApplySite applySite, ClassHierarchyAnalysis *cha, OptRemark::Emitter *ore, bool isMandatory) { LLVM_DEBUG(llvm::dbgs() << " Trying to devirtualize: " << *applySite.getInstruction()); @@ -1303,7 +1303,7 @@ swift::tryDevirtualizeApply(ApplySite applySite, ClassHierarchyAnalysis *cha, // %9 = apply %8(%6#1) : ... // if (isa(applySite.getCallee())) - return tryDevirtualizeWitnessMethod(applySite, ore, isMandatory); + return tryDevirtualizeWitnessMethod(pm, applySite, ore, isMandatory); // TODO: check if we can also de-virtualize partial applies of class methods. FullApplySite fas = FullApplySite::isa(applySite.getInstruction()); @@ -1332,7 +1332,7 @@ swift::tryDevirtualizeApply(ApplySite applySite, ClassHierarchyAnalysis *cha, auto *cd = classType.getClassOrBoundGenericClass(); if (isEffectivelyFinalMethod(fas, classType, cd, cha)) - return tryDevirtualizeClassMethod(fas, instance, cd, classType, ore, + return tryDevirtualizeClassMethod(pm, fas, instance, cd, classType, ore, true /*isEffectivelyFinalMethod*/); // Try to check if the exact dynamic type of the instance is statically @@ -1343,13 +1343,13 @@ swift::tryDevirtualizeApply(ApplySite applySite, ClassHierarchyAnalysis *cha, CanType classType = getSelfInstanceType(instance->getType().getASTType()); // This should never be null - make the check just to be on the safe side. if (ClassDecl *cd = classType.getClassOrBoundGenericClass()) - return tryDevirtualizeClassMethod(fas, instance, cd, classType, ore); + return tryDevirtualizeClassMethod(pm, fas, instance, cd, classType, ore); return {ApplySite(), false}; } if (auto exactTy = getExactDynamicType(cmi->getOperand(), cha)) { if (exactTy == cmi->getOperand()->getType()) - return tryDevirtualizeClassMethod(fas, cmi->getOperand(), cd, classType, + return tryDevirtualizeClassMethod(pm, fas, cmi->getOperand(), cd, classType, ore); } } @@ -1359,7 +1359,7 @@ swift::tryDevirtualizeApply(ApplySite applySite, ClassHierarchyAnalysis *cha, auto classType = getSelfInstanceType(instance->getType().getASTType()); auto *cd = classType.getClassOrBoundGenericClass(); - return tryDevirtualizeClassMethod(fas, instance, cd, classType, ore); + return tryDevirtualizeClassMethod(pm, fas, instance, cd, classType, ore); } return {ApplySite(), false}; diff --git a/lib/SILOptimizer/Utils/InstOptUtils.cpp b/lib/SILOptimizer/Utils/InstOptUtils.cpp index 64937a6f057f0..f364bdd86db05 100644 --- a/lib/SILOptimizer/Utils/InstOptUtils.cpp +++ b/lib/SILOptimizer/Utils/InstOptUtils.cpp @@ -36,6 +36,7 @@ #include "swift/SILOptimizer/OptimizerBridging.h" #include "swift/SILOptimizer/Utils/CFGOptUtils.h" #include "swift/SILOptimizer/Utils/DebugOptUtils.h" +#include "swift/SILOptimizer/Utils/OwnershipOptUtils.h" #include "swift/SILOptimizer/Utils/ValueLifetime.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringSwitch.h" @@ -175,6 +176,10 @@ bool swift::isInstructionTriviallyDead(SILInstruction *inst) { if (isa(inst)) return false; + // A dead borrowed-from can only be removed if the argument (= operand) is also removed. + if (isa(inst)) + return false; + // These invalidate enums so "write" memory, but that is not an essential // operation so we can remove these if they are trivially dead. if (isa(inst)) @@ -586,7 +591,8 @@ SILLinkage swift::getSpecializedLinkage(SILFunction *f, SILLinkage linkage) { /// handle all cases recognized by SILFunctionType::isABICompatibleWith (see /// areABICompatibleParamsOrReturns()). std::pair -swift::castValueToABICompatibleType(SILBuilder *builder, SILLocation loc, +swift::castValueToABICompatibleType(SILBuilder *builder, SILPassManager *pm, + SILLocation loc, SILValue value, SILType srcTy, SILType destTy, ArrayRef usePoints) { @@ -669,7 +675,7 @@ swift::castValueToABICompatibleType(SILBuilder *builder, SILLocation loc, // Cast the unwrapped value. SILValue castedUnwrappedValue; std::tie(castedUnwrappedValue, std::ignore) = castValueToABICompatibleType( - builder, loc, unwrappedValue, optionalSrcTy, optionalDestTy, usePoints); + builder, pm, loc, unwrappedValue, optionalSrcTy, optionalDestTy, usePoints); // Wrap into optional. An owned value is forwarded through the cast and into // the Optional. A borrowed value will have a nested borrow for the // rewrapped Optional. @@ -683,7 +689,9 @@ swift::castValueToABICompatibleType(SILBuilder *builder, SILLocation loc, builder->createBranch(loc, contBB, {noneValue}); builder->setInsertionPoint(contBB->begin()); - return {phi, true}; + updateBorrowedFromPhis(pm, { phi }); + + return {lookThroughBorrowedFromUser(phi), true}; } // Src is not optional, but dest is optional. @@ -697,7 +705,7 @@ swift::castValueToABICompatibleType(SILBuilder *builder, SILLocation loc, SILValue wrappedValue = builder->createOptionalSome(loc, value, loweredOptionalSrcType); // Cast the wrapped value. - return castValueToABICompatibleType(builder, loc, wrappedValue, + return castValueToABICompatibleType(builder, pm, loc, wrappedValue, wrappedValue->getType(), destTy, usePoints); } @@ -711,7 +719,7 @@ swift::castValueToABICompatibleType(SILBuilder *builder, SILLocation loc, // Cast the value if necessary. bool neededCFGChange; std::tie(element, neededCFGChange) = castValueToABICompatibleType( - builder, loc, element, srcTy.getTupleElementType(idx), + builder, pm, loc, element, srcTy.getTupleElementType(idx), destTy.getTupleElementType(idx), usePoints); changedCFG |= neededCFGChange; expectedTuple.push_back(element); diff --git a/lib/SILOptimizer/Utils/OwnershipOptUtils.cpp b/lib/SILOptimizer/Utils/OwnershipOptUtils.cpp index 38b1e6e046385..f68ef9522e8e4 100644 --- a/lib/SILOptimizer/Utils/OwnershipOptUtils.cpp +++ b/lib/SILOptimizer/Utils/OwnershipOptUtils.cpp @@ -29,6 +29,7 @@ #include "swift/SIL/SILArgument.h" #include "swift/SIL/SILBuilder.h" #include "swift/SIL/SILInstruction.h" +#include "swift/SILOptimizer/OptimizerBridging.h" #include "swift/SILOptimizer/Utils/CFGOptUtils.h" #include "swift/SILOptimizer/Utils/InstructionDeleter.h" #include "swift/SILOptimizer/Utils/ValueLifetime.h" @@ -1906,3 +1907,32 @@ bool swift::extendStoreBorrow(StoreBorrowInst *sbi, return true; } + +//===----------------------------------------------------------------------===// +// Swift Bridging +//===----------------------------------------------------------------------===// + +static BridgedUtilities::UpdateBorrowedFromFn updateBorrowedFromFunction; +static BridgedUtilities::UpdateBorrowedFromPhisFn updateBorrowedFromPhisFunction; + +void BridgedUtilities::registerBorrowedFromUpdater(UpdateBorrowedFromFn updateBorrowedFromFn, + UpdateBorrowedFromPhisFn updateBorrowedFromPhisFn) { + updateBorrowedFromFunction = updateBorrowedFromFn; + updateBorrowedFromPhisFunction = updateBorrowedFromPhisFn; +} + +void swift::updateBorrowedFrom(SILPassManager *pm, SILFunction *f) { + if (updateBorrowedFromFunction) + updateBorrowedFromFunction({pm->getSwiftPassInvocation()}, {f}); +} + +void swift::updateBorrowedFromPhis(SILPassManager *pm, ArrayRef phis) { + if (!updateBorrowedFromPhisFunction) + return; + + llvm::SmallVector bridgedPhis; + for (SILPhiArgument *phi : phis) { + bridgedPhis.push_back({phi}); + } + updateBorrowedFromPhisFunction({pm->getSwiftPassInvocation()}, ArrayRef(bridgedPhis)); +} diff --git a/lib/SILOptimizer/Utils/SILInliner.cpp b/lib/SILOptimizer/Utils/SILInliner.cpp index c422168994dfa..fde7cf5ae7e7c 100644 --- a/lib/SILOptimizer/Utils/SILInliner.cpp +++ b/lib/SILOptimizer/Utils/SILInliner.cpp @@ -889,6 +889,7 @@ InlineCost swift::instructionInlineCost(SILInstruction &I) { case SILInstructionKind::FixLifetimeInst: case SILInstructionKind::EndBorrowInst: case SILInstructionKind::BeginBorrowInst: + case SILInstructionKind::BorrowedFromInst: case SILInstructionKind::MarkDependenceInst: case SILInstructionKind::PreviousDynamicFunctionRefInst: case SILInstructionKind::DynamicFunctionRefInst: diff --git a/lib/Serialization/DeserializeSIL.cpp b/lib/Serialization/DeserializeSIL.cpp index 09cca5520bc90..499c3b6558a5d 100644 --- a/lib/Serialization/DeserializeSIL.cpp +++ b/lib/Serialization/DeserializeSIL.cpp @@ -2668,7 +2668,8 @@ bool SILDeserializer::readSILInstruction(SILFunction *Fn, ResultTy.getObjectType()); break; } - case SILInstructionKind::StructInst: { + case SILInstructionKind::StructInst: + case SILInstructionKind::BorrowedFromInst: { // Format: a type followed by a list of typed values. A typed value is // expressed by 4 IDs: TypeID, TypeCategory, ValueID, ValueResultNumber. auto Ty = MF->getType(TyID); @@ -2679,8 +2680,13 @@ bool SILDeserializer::readSILInstruction(SILFunction *Fn, Builder.maybeGetFunction(), ListOfValues[I + 2], getSILType(EltTy, (SILValueCategory)ListOfValues[I + 1], Fn))); } - ResultInst = Builder.createStruct( + if (OpCode == SILInstructionKind::StructInst) { + ResultInst = Builder.createStruct( Loc, getSILType(Ty, (SILValueCategory)TyCategory, Fn), OpList); + } else { + ResultInst = Builder.createBorrowedFrom( + Loc, OpList[0], ArrayRef(OpList).drop_front()); + } break; } case SILInstructionKind::TupleElementAddrInst: diff --git a/lib/Serialization/ModuleFormat.h b/lib/Serialization/ModuleFormat.h index d819e6a612878..310d106e615c0 100644 --- a/lib/Serialization/ModuleFormat.h +++ b/lib/Serialization/ModuleFormat.h @@ -58,7 +58,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0; /// describe what change you made. The content of this comment isn't important; /// it just ensures a conflict if two people change the module format. /// Don't worry about adhering to the 80-column limit for this line. -const uint16_t SWIFTMODULE_VERSION_MINOR = 869; // @implementation +const uint16_t SWIFTMODULE_VERSION_MINOR = 870; // borrowed-from instruction /// A standard hash seed used for all string hashes in a serialized module. /// diff --git a/lib/Serialization/SerializeSIL.cpp b/lib/Serialization/SerializeSIL.cpp index e34537a7083b5..8a11019c66073 100644 --- a/lib/Serialization/SerializeSIL.cpp +++ b/lib/Serialization/SerializeSIL.cpp @@ -2345,12 +2345,13 @@ void SILSerializer::writeSILInstruction(const SILInstruction &SI) { RTAI->getOperand()); break; } - case SILInstructionKind::StructInst: { + case SILInstructionKind::StructInst: + case SILInstructionKind::BorrowedFromInst: { // Format: a type followed by a list of typed values. A typed value is // expressed by 4 IDs: TypeID, TypeCategory, ValueID, ValueResultNumber. - const StructInst *StrI = cast(&SI); + const auto *svi = cast(&SI); SmallVector ListOfValues; - for (auto Elt : StrI->getElements()) { + for (auto Elt : svi->getOperandValues()) { ListOfValues.push_back(S.addTypeRef(Elt->getType().getRawASTType())); ListOfValues.push_back((unsigned)Elt->getType().getCategory()); ListOfValues.push_back(addValueRef(Elt)); @@ -2358,8 +2359,8 @@ void SILSerializer::writeSILInstruction(const SILInstruction &SI) { SILOneTypeValuesLayout::emitRecord( Out, ScratchRecord, SILAbbrCodes[SILOneTypeValuesLayout::Code], - (unsigned)SI.getKind(), S.addTypeRef(StrI->getType().getRawASTType()), - (unsigned)StrI->getType().getCategory(), ListOfValues); + (unsigned)SI.getKind(), S.addTypeRef(svi->getType().getRawASTType()), + (unsigned)svi->getType().getCategory(), ListOfValues); break; } case SILInstructionKind::TupleElementAddrInst: diff --git a/test/SIL/OwnershipVerifier/arguments.sil b/test/SIL/OwnershipVerifier/arguments.sil index eb2c17e5f383f..bb6b3a5617104 100644 --- a/test/SIL/OwnershipVerifier/arguments.sil +++ b/test/SIL/OwnershipVerifier/arguments.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -module-name Swift -sil-ownership-verifier-enable-testing -ownership-verifier-textual-error-dumper -enable-sil-verify-all=0 %s -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -module-name Swift -disable-swift-verification -sil-ownership-verifier-enable-testing -ownership-verifier-textual-error-dumper -enable-sil-verify-all=0 %s -o /dev/null 2>&1 | %FileCheck %s // REQUIRES: asserts // This is a test that verifies ownership behavior around arguments that should diff --git a/test/SIL/OwnershipVerifier/borrow_cast_validate.sil b/test/SIL/OwnershipVerifier/borrow_cast_validate.sil index 6c378c522fea2..4dead34dd4e22 100644 --- a/test/SIL/OwnershipVerifier/borrow_cast_validate.sil +++ b/test/SIL/OwnershipVerifier/borrow_cast_validate.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-ownership-verifier-enable-testing -ownership-verifier-textual-error-dumper -enable-sil-verify-all=0 %s -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -sil-ownership-verifier-enable-testing -disable-swift-verification -ownership-verifier-textual-error-dumper -enable-sil-verify-all=0 %s -o /dev/null 2>&1 | %FileCheck %s // REQUIRES: asserts import Builtin diff --git a/test/SIL/OwnershipVerifier/borrow_extract_validate.sil b/test/SIL/OwnershipVerifier/borrow_extract_validate.sil index 5dd5284bc37e0..f4f489ef62719 100644 --- a/test/SIL/OwnershipVerifier/borrow_extract_validate.sil +++ b/test/SIL/OwnershipVerifier/borrow_extract_validate.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-ownership-verifier-enable-testing -ownership-verifier-textual-error-dumper -enable-sil-verify-all=0 %s -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -disable-swift-verification -sil-ownership-verifier-enable-testing -ownership-verifier-textual-error-dumper -enable-sil-verify-all=0 %s -o /dev/null 2>&1 | %FileCheck %s // REQUIRES: asserts import Builtin diff --git a/test/SIL/OwnershipVerifier/borrow_scope_introducing_operands_positive.sil b/test/SIL/OwnershipVerifier/borrow_scope_introducing_operands_positive.sil index e132a058d34b4..29b365ca17964 100644 --- a/test/SIL/OwnershipVerifier/borrow_scope_introducing_operands_positive.sil +++ b/test/SIL/OwnershipVerifier/borrow_scope_introducing_operands_positive.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-sil-verify-all=0 -o /dev/null %s 2>&1 +// RUN: %target-sil-opt -update-borrowed-from -enable-sil-verify-all=0 -o /dev/null %s 2>&1 // REQUIRES: asserts diff --git a/test/SIL/OwnershipVerifier/borrow_validate.sil b/test/SIL/OwnershipVerifier/borrow_validate.sil index b2c8786024fa1..04496e3e8a44c 100644 --- a/test/SIL/OwnershipVerifier/borrow_validate.sil +++ b/test/SIL/OwnershipVerifier/borrow_validate.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-ownership-verifier-enable-testing -ownership-verifier-textual-error-dumper -enable-sil-verify-all=0 %s -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -disable-swift-verification -sil-ownership-verifier-enable-testing -ownership-verifier-textual-error-dumper -enable-sil-verify-all=0 %s -o /dev/null 2>&1 | %FileCheck %s // REQUIRES: asserts import Builtin diff --git a/test/SIL/OwnershipVerifier/guaranteed_phis.sil b/test/SIL/OwnershipVerifier/guaranteed_phis.sil index f0433eb19c723..f0a3c2a534537 100644 --- a/test/SIL/OwnershipVerifier/guaranteed_phis.sil +++ b/test/SIL/OwnershipVerifier/guaranteed_phis.sil @@ -1,4 +1,5 @@ -// RUN: %target-sil-opt -enable-sil-verify-all %s +// RUN: %target-sil-opt -update-borrowed-from -enable-sil-verify-all %s + import Swift import Builtin diff --git a/test/SIL/OwnershipVerifier/guaranteed_phis_errors.sil b/test/SIL/OwnershipVerifier/guaranteed_phis_errors.sil index 0fa004e628f5c..4236e4a63df06 100644 --- a/test/SIL/OwnershipVerifier/guaranteed_phis_errors.sil +++ b/test/SIL/OwnershipVerifier/guaranteed_phis_errors.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -module-name Swift -sil-ownership-verifier-enable-testing -ownership-verifier-textual-error-dumper -enable-sil-verify-all=0 %s -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -disable-swift-verification -module-name Swift -sil-ownership-verifier-enable-testing -ownership-verifier-textual-error-dumper -enable-sil-verify-all=0 %s -o /dev/null 2>&1 | %FileCheck %s class SuperKlass {} diff --git a/test/SIL/OwnershipVerifier/load_borrow_verify.sil b/test/SIL/OwnershipVerifier/load_borrow_verify.sil index 2e2eb92ab8396..093e74a175355 100644 --- a/test/SIL/OwnershipVerifier/load_borrow_verify.sil +++ b/test/SIL/OwnershipVerifier/load_borrow_verify.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-sil-verify-all %s | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -enable-sil-verify-all %s | %FileCheck %s // Test that the LoadBorrowImmutabilityChecker accepts these cases. // The verification should not bail-out on these, but there's no way diff --git a/test/SIL/OwnershipVerifier/load_borrow_verify_errors.sil b/test/SIL/OwnershipVerifier/load_borrow_verify_errors.sil index 4c11fd927cbdd..9ec37d52b6a9f 100644 --- a/test/SIL/OwnershipVerifier/load_borrow_verify_errors.sil +++ b/test/SIL/OwnershipVerifier/load_borrow_verify_errors.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt %s -verify-continue-on-failure=true -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt %s -disable-swift-verification -verify-continue-on-failure=true -o /dev/null 2>&1 | %FileCheck %s import Builtin class Klass {} diff --git a/test/SIL/OwnershipVerifier/reborrowflag.sil b/test/SIL/OwnershipVerifier/reborrowflag.sil index 53f568f5b73e5..285e666fc888c 100644 --- a/test/SIL/OwnershipVerifier/reborrowflag.sil +++ b/test/SIL/OwnershipVerifier/reborrowflag.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt %s -disable-populate-ownership-flags -verify-continue-on-failure=true -sil-ownership-verifier-enable-testing -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt %s -update-borrowed-from -disable-populate-ownership-flags -verify-continue-on-failure=true -sil-ownership-verifier-enable-testing -o /dev/null 2>&1 | %FileCheck %s sil_stage canonical class Klass {} diff --git a/test/SIL/OwnershipVerifier/use_verifier.sil b/test/SIL/OwnershipVerifier/use_verifier.sil index ea76f78ba68df..7e466d699cc9f 100644 --- a/test/SIL/OwnershipVerifier/use_verifier.sil +++ b/test/SIL/OwnershipVerifier/use_verifier.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-objc-interop -enable-sil-verify-all=0 -module-name Swift -o /dev/null %s 2>&1 +// RUN: %target-sil-opt -update-borrowed-from -enable-objc-interop -enable-sil-verify-all=0 -module-name Swift -o /dev/null %s 2>&1 // REQUIRES: asserts // This file is meant to contain tests that previously the verifier treated diff --git a/test/SIL/Parser/borrow.sil b/test/SIL/Parser/borrow.sil index bc113da8a1397..0b640f279d77c 100644 --- a/test/SIL/Parser/borrow.sil +++ b/test/SIL/Parser/borrow.sil @@ -30,6 +30,11 @@ bb0(%0 : $*Builtin.NativeObject, %1 : @guaranteed $Builtin.NativeObject): class C {} +enum FakeOptional { + case none + case some(T) +} + // CHECK-LABEL: sil [ossa] @foo // CHECK: begin_borrow {{%[^,]+}} // CHECK: begin_borrow [lexical] {{%[^,]+}} @@ -49,8 +54,9 @@ sil [ossa] @foo : $@convention(thin) () -> () { } // CHECK-LABEL: sil [ossa] @reborrowTest : $@convention(thin) (@owned C) -> () { -// CHECK: bb3([[ARG:%.*]] : @reborrow @guaranteed $C): -// CHECK: } // end sil function 'reborrowTest' +// CHECK: bb3([[ARG:%.*]] : @reborrow @guaranteed $C): +// CHECK-NEXT: {{%.*}} = borrowed [[ARG]] : $C from (%0 : $C) +// CHECK: } // end sil function 'reborrowTest' sil [ossa] @reborrowTest : $@convention(thin) (@owned C) -> () { bb0(%0 : @owned $C): cond_br undef, bb1, bb2 @@ -64,9 +70,24 @@ bb2: br bb3(%b2 : $C) bb3(%r : @reborrow @guaranteed $C): - end_borrow %r : $C + %f = borrowed %r : $C from (%0 : $C) + end_borrow %f : $C destroy_value %0 : $C %9999 = tuple() return %9999 : $() } +// CHECK-LABEL: sil [ossa] @reborrowNone : +// CHECK: bb1([[ARG:%.*]] : @guaranteed $FakeOptional): +// CHECK-NEXT: {{%.*}} = borrowed [[ARG]] : $FakeOptional from () +// CHECK: } // end sil function 'reborrowNone' +sil [ossa] @reborrowNone : $@convention(thin) () -> () { +bb0: + %0 = enum $FakeOptional, #FakeOptional.none!enumelt + br bb1(%0 : $FakeOptional) + +bb1(%2 : @guaranteed $FakeOptional): + %3 = borrowed %2 : $FakeOptional from () + %4 = tuple () + return %4 : $() +} diff --git a/test/SIL/Parser/borrow_argument.sil b/test/SIL/Parser/borrow_argument.sil index 914e82a2ae5da..a1764b904feec 100644 --- a/test/SIL/Parser/borrow_argument.sil +++ b/test/SIL/Parser/borrow_argument.sil @@ -1,19 +1,23 @@ // RUN: %target-sil-opt %s | %target-sil-opt | %FileCheck %s +// REQUIRES: swift_in_compiler + sil_stage canonical import Builtin // CHECK-LABEL: sil [ossa] @borrow_argument_test : $@convention(thin) (@guaranteed Builtin.NativeObject) -> () { // CHECK: bb1([[PHIBBARG:%.*]] : @reborrow @guaranteed $Builtin.NativeObject): -// CHECK: end_borrow [[PHIBBARG]] : $Builtin.NativeObject +// CHECK: [[BF:%.*]] = borrowed [[PHIBBARG]] : $Builtin.NativeObject from (%0 : $Builtin.NativeObject) +// CHECK: end_borrow [[BF]] : $Builtin.NativeObject sil [ossa] @borrow_argument_test : $@convention(thin) (@guaranteed Builtin.NativeObject) -> () { bb0(%0 : @guaranteed $Builtin.NativeObject): %1 = begin_borrow %0 : $Builtin.NativeObject br bb1(%1 : $Builtin.NativeObject) bb1(%2 : @guaranteed $Builtin.NativeObject): - end_borrow %2 : $Builtin.NativeObject + %2f = borrowed %2 : $Builtin.NativeObject from (%0 : $Builtin.NativeObject) + end_borrow %2f : $Builtin.NativeObject %4 = tuple() return %4 : $() } diff --git a/test/SIL/Parser/ownership_arguments.sil b/test/SIL/Parser/ownership_arguments.sil index eca20b4e68255..bb6fe007bbcd1 100644 --- a/test/SIL/Parser/ownership_arguments.sil +++ b/test/SIL/Parser/ownership_arguments.sil @@ -14,8 +14,9 @@ bb0(%0 : @owned $Builtin.NativeObject, %1 : @unowned $Builtin.NativeObject, %2 : br bb1(%0 : $Builtin.NativeObject, %1 : $Builtin.NativeObject, %2a : $Builtin.NativeObject, %3 : $Builtin.Int32) bb1(%4 : @owned $Builtin.NativeObject, %5 : @unowned $Builtin.NativeObject, %6 : @guaranteed $Builtin.NativeObject, %7 : $Builtin.Int32): + %6f = borrowed %6 : $Builtin.NativeObject from (%2 : $Builtin.NativeObject) destroy_value %4 : $Builtin.NativeObject - end_borrow %6 : $Builtin.NativeObject + end_borrow %6f : $Builtin.NativeObject %9999 = tuple() return %9999 : $() } diff --git a/test/SIL/Serialization/borrow_argument.sil b/test/SIL/Serialization/borrow_argument.sil index 8a5e80f251b06..4b37c0421e1b4 100644 --- a/test/SIL/Serialization/borrow_argument.sil +++ b/test/SIL/Serialization/borrow_argument.sil @@ -10,14 +10,16 @@ import Builtin // CHECK-LABEL: sil [serialized] [ossa] @borrow_argument_test : $@convention(thin) (@guaranteed Builtin.NativeObject) -> () { // CHECK: bb1([[PHIBBARG:%.*]] : @reborrow @guaranteed $Builtin.NativeObject): -// CHECK: end_borrow [[PHIBBARG]] : $Builtin.NativeObject +// CHECK: [[BF:%.*]] = borrowed [[PHIBBARG]] : $Builtin.NativeObject from (%0 : $Builtin.NativeObject) +// CHECK: end_borrow [[BF]] : $Builtin.NativeObject sil [serialized] [ossa] @borrow_argument_test : $@convention(thin) (@guaranteed Builtin.NativeObject) -> () { bb0(%0 : @guaranteed $Builtin.NativeObject): %0a = begin_borrow %0 : $Builtin.NativeObject br bb1(%0a : $Builtin.NativeObject) bb1(%1 : @guaranteed $Builtin.NativeObject): - end_borrow %1 : $Builtin.NativeObject + %1f = borrowed %1 : $Builtin.NativeObject from (%0 : $Builtin.NativeObject) + end_borrow %1f : $Builtin.NativeObject %4 = tuple() return %4 : $() } diff --git a/test/SIL/Serialization/ownershipflags.sil b/test/SIL/Serialization/ownershipflags.sil index 8331cfadb8353..c8fe139b5561b 100644 --- a/test/SIL/Serialization/ownershipflags.sil +++ b/test/SIL/Serialization/ownershipflags.sil @@ -60,7 +60,8 @@ bb2: br bb3(%b2 : $Klass) bb3(%r : @reborrow @guaranteed $Klass): - end_borrow %r : $Klass + %rf = borrowed %r : $Klass from (%0 : $Klass) + end_borrow %rf : $Klass destroy_value %0 : $Klass %9999 = tuple() return %9999 : $() diff --git a/test/SIL/memory_lifetime_failures.sil b/test/SIL/memory_lifetime_failures.sil index 83e507c68388b..e36bcc9dc12de 100644 --- a/test/SIL/memory_lifetime_failures.sil +++ b/test/SIL/memory_lifetime_failures.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -dont-abort-on-memory-lifetime-errors -o /dev/null %s 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -disable-swift-verification -dont-abort-on-memory-lifetime-errors -o /dev/null %s 2>&1 | %FileCheck %s // REQUIRES: asserts // REQUIRES: swift_in_compiler diff --git a/test/SILOptimizer/accesspath_uses_ossa.sil b/test/SILOptimizer/accesspath_uses_ossa.sil index afce86502364b..0eec62821ca2f 100644 --- a/test/SILOptimizer/accesspath_uses_ossa.sil +++ b/test/SILOptimizer/accesspath_uses_ossa.sil @@ -1,5 +1,5 @@ -// RUN: %target-sil-opt %s -access-storage-dump -enable-access-storage-dump-uses -enable-sil-verify-all -o /dev/null | %FileCheck %s -// RUN: %target-sil-opt %s -access-path-verification -o /dev/null +// RUN: %target-sil-opt %s -disable-swift-verification -access-storage-dump -enable-access-storage-dump-uses -enable-sil-verify-all -o /dev/null | %FileCheck %s +// RUN: %target-sil-opt %s -disable-swift-verification -access-path-verification -o /dev/null // REQUIRES: PTRSIZE=64 diff --git a/test/SILOptimizer/borrow_introducer_unit.sil b/test/SILOptimizer/borrow_introducer_unit.sil index 00fcd8c387fbc..a084eb09095a1 100644 --- a/test/SILOptimizer/borrow_introducer_unit.sil +++ b/test/SILOptimizer/borrow_introducer_unit.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -test-runner %s -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -test-runner -sil-disable-input-verify %s -o /dev/null 2>&1 | %FileCheck %s // // REQUIRES: swift_in_compiler @@ -167,7 +167,7 @@ exit: // CHECK-LABEL: introducer_single: borrow_introducers with: %aggregate2 // CHECK: } // end sil function 'introducer_single' -// CHECK: Borrow introducers for: %11 = struct $PairC (%9 : $C, %10 : $C) +// CHECK: Borrow introducers for: %12 = struct $PairC (%10 : $C, %11 : $C) // CHECK-NEXT: %0 = argument of bb0 : $C // CHECK-NEXT: introducer_single: borrow_introducers with: %aggregate2 sil [ossa] @introducer_single : $@convention(thin) (@guaranteed C) -> () { @@ -217,8 +217,8 @@ exit: // CHECK: bb1([[REBORROW:%.*]] : @reborrow @guaranteed $C // CHECK: } // end sil function 'introducer_revisit_reborrow' // CHECK: Borrow introducers for: %{{.*}} = struct $PairC (%{{.*}} : $C, %{{.*}} : $C) -// CHECK-NEXT: [[REBORROW]] = argument of bb1 : $C // CHECK-NEXT: %0 = argument of bb0 : $C +// CHECK-NEXT: [[REBORROW]] = argument of bb1 : $C // CHECK-NEXT: introducer_revisit_reborrow: borrow_introducers with: %aggregate2 sil [ossa] @introducer_revisit_reborrow : $@convention(thin) (@guaranteed C) -> () { entry(%0 : @guaranteed $C): @@ -259,9 +259,9 @@ exit: // CHECK: bb1([[REBORROW:%.*]] : @reborrow @guaranteed $C // CHECK: } // end sil function 'introducer_multiple_borrow' // CHECK: Borrow introducers for: %{{.*}} = struct $PairC (%{{.*}} : $C, %{{.*}} : $C) -// CHECK-NEXT: [[REBORROW]] = argument of bb1 : $C // CHECK-NEXT: %0 = argument of bb0 : $C // CHECK-NEXT: [[BORROW2]] = begin_borrow %0 : $C +// CHECK-NEXT: [[REBORROW]] = argument of bb1 : $C // CHECK-NEXT: introducer_multiple_borrow: borrow_introducers with: %aggregate2 sil [ossa] @introducer_multiple_borrow : $@convention(thin) (@guaranteed C) -> () { entry(%0 : @guaranteed $C): @@ -347,8 +347,8 @@ bb2: // CHECK-LABEL: introducer_example: borrow_introducers with: %first // CHECK: Borrow introducers for: %{{.*}} = struct_extract %{{.*}} : $PairC, #PairC.first -// CHECK-NEXT: %{{.*}} = begin_borrow %0 : $C // CHECK-NEXT: %{{.*}} = argument of bb0 : $C +// CHECK-NEXT: %{{.*}} = begin_borrow %0 : $C // CHECK-NEXT: introducer_example: borrow_introducers with: %first // CHECK-LABEL: introducer_example: borrow_introducers with: %load @@ -379,8 +379,8 @@ bb0(%0 : @owned $C, // CHECK-NEXT: testSelfIntroducer: borrow_introducers with: %phi // CHECK-LABEL: testSelfIntroducer: enclosing_values with: %reborrow -// CHECK: Enclosing values for: %4 = argument of bb1 : $C // users: %8, %7 -// CHECK-NEXT: %0 = argument of bb0 : $C // user: %1 +// CHECK: Enclosing values for: %4 = argument of bb1 : $C +// CHECK-NEXT: %0 = argument of bb0 : $C // CHECK-NEXT: testSelfIntroducer: enclosing_values with: %reborrow sil [ossa] @testSelfIntroducer : $@convention(thin) (@guaranteed C) -> () { bb0(%0 : @guaranteed $C): @@ -409,7 +409,7 @@ bb3: // one of the outer adjacent phis as an enclosing value for %inner. // // CHECK-LABEL: testEnclosingRevisitReborrow: enclosing_values with: %inner -// CHECK: Enclosing values for: %10 = argument of bb4 : $C // user: %11 +// CHECK: Enclosing values for: %11 = argument of bb4 : $C // CHECK-NEXT: %{{.*}} = argument of bb4 : $C // CHECK-NEXT: %{{.*}} = argument of bb4 : $C // CHECK-NEXT: end running test 1 of 1 on testEnclosingRevisitReborrow: enclosing_values with: %inner diff --git a/test/SILOptimizer/borrowed_from_updater.sil b/test/SILOptimizer/borrowed_from_updater.sil new file mode 100644 index 0000000000000..f2b6c0a7bbbaa --- /dev/null +++ b/test/SILOptimizer/borrowed_from_updater.sil @@ -0,0 +1,288 @@ +// RUN: %target-sil-opt -update-borrowed-from -sil-disable-input-verify %s | %FileCheck %s +// +// REQUIRES: swift_in_compiler + +sil_stage raw + +import Builtin + +struct Trivial { + var value: Builtin.Int32 +} + +enum FakeOptional { +case none +case some(T) +} + +struct PairC { + var first: C + var second: C +} + +class C { + var field: C +} + +class D : C {} + +sil @coro : $@yield_once @convention(thin) () -> @yields @guaranteed C + +// The introducer of an introducer is always itself. + +// CHECK-LABEL: sil [ossa] @introducer_identity : +// CHECK: bb1([[A:%.*]] : @reborrow @guaranteed $C): +// CHECK-NEXT: [[B:%.*]] = borrowed [[A]] : $C from (%0 : $C) +// CHECK-NEXT: end_borrow [[B]] +// CHECK: } // end sil function 'introducer_identity' +sil [ossa] @introducer_identity : $@convention(thin) (@guaranteed C, @in C) -> () { +bb0(%0 : @guaranteed $C, %1 : $*C): + %borrow1 = begin_borrow %0 : $C + br bb1(%borrow1 : $C) + +bb1(%reborrow : @guaranteed $C): + end_borrow %reborrow : $C + destroy_addr %1 : $*C + %retval = tuple () + return %retval : $() +} + +// There is no introducer if the guaranteed value is produced from a +// trivial value. + +// CHECK-LABEL: sil [ossa] @introducer_trivial : +// CHECK: bb1([[A:%.*]] : @guaranteed $FakeOptional): +// CHECK-NEXT: [[B:%.*]] = borrowed [[A]] : $FakeOptional from () +// CHECK: } // end sil function 'introducer_trivial' +sil [ossa] @introducer_trivial : $@convention(thin) () -> () { +bb0: + %trivial = enum $FakeOptional, #FakeOptional.none!enumelt + br bb1(%trivial : $FakeOptional) + +bb1(%phi : @guaranteed $FakeOptional): + %retval = tuple () + return %retval : $() +} + +// An unreachable phi currently looks like a reborrow. +// +// TODO: When we have a reborrow flag, an unreachable phi can be +// either a reborrow or forwarded, as long as it has no end_borrow. + +// CHECK-LABEL: sil [ossa] @introducer_unreachable : $@convention(thin) () -> () { +// CHECK: bb1([[A:%.*]] : @reborrow @guaranteed $C): +// CHECK-NEXT: [[B:%.*]] = borrowed [[A]] : $C from () +// CHECK: } // end sil function 'introducer_unreachable' +sil [ossa] @introducer_unreachable : $@convention(thin) () -> () { +bb0: + br bb2 + +bb1(%phiCycle : @guaranteed $C): + br bb1(%phiCycle : $C) + +bb2: + %retval = tuple () + return %retval : $() +} + +// %reborrow introduces multiple borrow scopes. But it should only appear +// in the list once. + +// The forwarding %phi originates from %borrow1 and %0. But +// %borrow1 cannot be an introducer in bb2 because it's scope ends at +// %reborrow. Therefore, %reborrow is an introducer from separate phi +// paths, but should only appear in the introducer list once. +// +// CHECK-LABEL: sil [ossa] @introducer_revisit_reborrow : +// CHECK: bb1([[A1:%.*]] : @reborrow @guaranteed $C, [[A2:%.*]] : @guaranteed $PairC): +// CHECK-NEXT: [[B2:%.*]] = borrowed [[A2]] : $PairC from ([[A1]] : $C, %0 : $C) +// CHECK-NEXT: [[B1:%.*]] = borrowed [[A1]] : $C from (%0 : $C) +// CHECK: } // end sil function 'introducer_revisit_reborrow' +sil [ossa] @introducer_revisit_reborrow : $@convention(thin) (@guaranteed C) -> () { +bb0(%0 : @guaranteed $C): + %borrow1 = begin_borrow %0 : $C + %aggregate = struct $PairC(%0 : $C, %borrow1 : $C) + br bb1(%borrow1 : $C, %aggregate : $PairC) + +bb1(%reborrow : @guaranteed $C, %phi : @guaranteed $PairC): + %first = struct_extract %phi : $PairC, #PairC.first + %aggregate2 = struct $PairC(%reborrow : $C, %first : $C) + br bb2 + +bb2: + end_borrow %reborrow : $C + %retval = tuple () + return %retval : $() +} + +// %phi originates from %0, %borrow1, & %borrow2. %borrow1 is, however, +// reborrowed in bb1. + +// CHECK-LABEL: sil [ossa] @introducer_multiple_borrow : +// CHECK: [[BB1:%.*]] = begin_borrow %0 +// CHECK: [[BB2:%.*]] = begin_borrow %0 +// CHECK: bb1([[A1:%.*]] : @reborrow @guaranteed $C, [[A2:%.*]] : @guaranteed $PairC): +// CHECK-NEXT: [[B2:%.*]] = borrowed [[A2]] : $PairC from ([[BB2]] : $C, %0 : $C) +// CHECK-NEXT: [[B1:%.*]] = borrowed [[A1]] : $C from (%0 : $C) +// CHECK: } // end sil function 'introducer_multiple_borrow' +sil [ossa] @introducer_multiple_borrow : $@convention(thin) (@guaranteed C) -> () { +bb0(%0 : @guaranteed $C): + %borrow1 = begin_borrow %0 : $C + %borrow2 = begin_borrow %0 : $C + %aggregate = struct $PairC(%0 : $C, %borrow2 : $C) + br bb1(%borrow1 : $C, %aggregate : $PairC) + +bb1(%reborrow : @guaranteed $C, %phi : @guaranteed $PairC): + %first = struct_extract %phi : $PairC, #PairC.first + %aggregate2 = struct $PairC(%reborrow : $C, %first : $C) + br bb2 + +bb2: + end_borrow %reborrow : $C + end_borrow %borrow2 : $C + %retval = tuple () + return %retval : $() +} + +// CHECK-LABEL: sil [ossa] @testSelfIntroducer : +// CHECK: bb1([[A1:%.*]] : @reborrow @guaranteed $C, [[A2:%.*]] : @guaranteed $D): +// CHECK-NEXT: [[B2:%.*]] = borrowed [[A2]] : $D from ([[A1]] : $C) +// CHECK-NEXT: [[B1:%.*]] = borrowed [[A1]] : $C from (%0 : $C) +// CHECK: } // end sil function 'testSelfIntroducer' +sil [ossa] @testSelfIntroducer : $@convention(thin) (@guaranteed C) -> () { +bb0(%0 : @guaranteed $C): + %borrow = begin_borrow %0 : $C + %d = unconditional_checked_cast %borrow : $C to D + br bb1(%borrow : $C, %d : $D) + +bb1(%reborrow : @guaranteed $C, %phi : @guaranteed $D): + cond_br undef, bb2, bb3 + +bb2: + br bb1(%reborrow : $C, %phi : $D) + +bb3: + end_borrow %reborrow : $C + %99 = tuple() + return %99 : $() +} + +// Test the reborrow cache in EnclosingValues. Here, %reborrow must be +// visited once on each path, and each time the recursive algorithm +// must find the enclosing def %0, which maps to a %outer0 on one path +// and %outer1 on another path. If the cache fails, then we only find +// one of the outer adjacent phis as and enclosing value for %inner. +// +// CHECK-LABEL: sil [ossa] @testEnclosingRevisitReborrow : +// CHECK: bb1([[A1:%.*]] : @reborrow @guaranteed $C): +// CHECK-NEXT: [[B1:%.*]] = borrowed [[A1]] : $C from (%0 : $C) +// CHECK: bb4([[A2:%.*]] : @owned $C, [[A3:%.*]] : @owned $C, [[A4:%.*]] : @reborrow @guaranteed $C): +// CHECK-NEXT: [[B2:%.*]] = borrowed [[A4]] : $C from ([[A3]] : $C, [[A2]] : $C) +// CHECK: } // end sil function 'testEnclosingRevisitReborrow' +sil [ossa] @testEnclosingRevisitReborrow : $@convention(thin) (@owned C, @owned C) -> () { +bb0(%0 : @owned $C, %1 : @owned $C): + %borrow = begin_borrow %0 : $C + br bb1(%borrow : $C) + +bb1(%reborrow : @guaranteed $C): + cond_br undef, bb2, bb3 + +bb2: + br bb4(%0 : $C, %1 : $C, %reborrow : $C) + +bb3: + br bb4(%1 : $C, %0 : $C, %reborrow : $C) + +bb4(%outer0 : @owned $C, %outer1 : @owned $C, %inner : @guaranteed $C): + end_borrow %inner : $C + destroy_value %outer0 : $C + destroy_value %outer1 : $C + %99 = tuple() + return %99 : $() +} + +// CHECK-LABEL: sil [ossa] @testLoadBorrow : +// CHECK: bb1([[A1:%.*]] : @reborrow @guaranteed $C): +// CHECK-NEXT: [[B1:%.*]] = borrowed [[A1]] : $C from () +// CHECK: } // end sil function 'testLoadBorrow' +sil [ossa] @testLoadBorrow : $@convention(thin) (@in_guaranteed C) -> () { +bb0(%0 : $*C): + %2 = load_borrow %0 : $*C + br bb1(%2 : $C) + +bb1(%3 : @guaranteed $C): + end_borrow %3 : $C + %99 = tuple() + return %99 : $() +} + +// CHECK-LABEL: sil [ossa] @testTwoLevels : +// CHECK: bb1([[A1:%.*]] : @reborrow @guaranteed $C): +// CHECK-NEXT: [[B1:%.*]] = borrowed [[A1]] : $C from (%0 : $C) +// CHECK: bb2([[A2:%.*]] : @reborrow @guaranteed $C): +// CHECK-NEXT: [[B2:%.*]] = borrowed [[A2]] : $C from (%0 : $C) +// CHECK: } // end sil function 'testTwoLevels' +sil [ossa] @testTwoLevels : $@convention(thin) (@owned C) -> () { +bb0(%0 : @owned $C): + %2 = begin_borrow %0 : $C + br bb1(%2 : $C) + +bb1(%3 : @reborrow @guaranteed $C): + br bb2(%3 : $C) + +bb2(%6 : @reborrow @guaranteed $C): + end_borrow %6 : $C + destroy_value %0 : $C + %99 = tuple() + return %99 : $() +} + +// CHECK-LABEL: sil [ossa] @testTwoLevelsWithPair : +// CHECK: bb1([[A1:%.*]] : @reborrow @guaranteed $C): +// CHECK-NEXT: [[B1:%.*]] = borrowed [[A1]] : $C from (%1 : $C) +// CHECK: bb2([[A2:%.*]] : @reborrow @guaranteed $C): +// CHECK-NEXT: [[B2:%.*]] = borrowed [[A2]] : $C from (%1 : $C) +// CHECK: } // end sil function 'testTwoLevelsWithPair' +sil [ossa] @testTwoLevelsWithPair : $@convention(thin) (@owned PairC) -> () { +bb0(%0 : @owned $PairC): + (%1, %2) = destructure_struct %0 : $PairC + %3 = begin_borrow %1 : $C + br bb1(%3 : $C) + +bb1(%4 : @reborrow @guaranteed $C): + br bb2(%4 : $C) + +bb2(%6 : @reborrow @guaranteed $C): + end_borrow %6 : $C + destroy_value %1 : $C + destroy_value %2 : $C + %99 = tuple() + return %99 : $() +} + +// CHECK-LABEL: sil [ossa] @testTwoLevelReborrow : +// CHECK: bb1([[A1:%.*]] : @reborrow @guaranteed $PairC, [[A2:%.*]] : @guaranteed $C): +// CHECK-NEXT: [[B2:%.*]] = borrowed [[A2]] : $C from ([[A1]] : $PairC) +// CHECK-NEXT: [[B1:%.*]] = borrowed [[A1]] : $PairC from (%0 : $PairC) +// CHECK-NEXT: br bb2([[B1]] : $PairC, [[B2]] : $C) +// CHECK: bb2([[A3:%.*]] : @reborrow @guaranteed $PairC, [[A4:%.*]] : @guaranteed $C): +// CHECK-NEXT: [[B4:%.*]] = borrowed [[A4]] : $C from ([[A3]] : $PairC) +// CHECK-NEXT: [[B3:%.*]] = borrowed [[A3]] : $PairC from (%0 : $PairC) +// CHECK-NEXT: end_borrow [[B3]] +// CHECK: } // end sil function 'testTwoLevelReborrow' +sil [ossa] @testTwoLevelReborrow : $@convention(thin) (@owned PairC) -> () { +bb0(%0 : @owned $PairC): + %2 = begin_borrow %0 : $PairC + %3 = struct_extract %2 : $PairC, #PairC.first + br bb1(%2 : $PairC, %3 : $C) + +bb1(%5 : @reborrow @guaranteed $PairC, %6 : @guaranteed $C): + br bb2(%5 : $PairC, %6 : $C) + +bb2(%8 : @reborrow @guaranteed $PairC, %9 : @guaranteed $C): + end_borrow %8 : $PairC + destroy_value %0 : $PairC + %99 = tuple() + return %99 : $() +} + diff --git a/test/SILOptimizer/conditionforwarding_nontrivial_ossa.sil b/test/SILOptimizer/conditionforwarding_nontrivial_ossa.sil index a255b6899b1bd..223c854eec1eb 100644 --- a/test/SILOptimizer/conditionforwarding_nontrivial_ossa.sil +++ b/test/SILOptimizer/conditionforwarding_nontrivial_ossa.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-debuginfo -enable-sil-verify-all %s -condition-forwarding | %FileCheck %s +// RUN: %target-sil-opt -sil-print-debuginfo -update-borrowed-from -enable-sil-verify-all %s -condition-forwarding | %FileCheck %s import Builtin import Swift diff --git a/test/SILOptimizer/copy_propagation.sil b/test/SILOptimizer/copy_propagation.sil index e210de9f320ce..84b9b58a044c5 100644 --- a/test/SILOptimizer/copy_propagation.sil +++ b/test/SILOptimizer/copy_propagation.sil @@ -1,5 +1,5 @@ -// RUN: %target-sil-opt -copy-propagation -canonical-ossa-rewrite-borrows -enable-sil-verify-all %s | %FileCheck %s --check-prefixes=CHECK,CHECK-OPT -// RUN: %target-sil-opt -mandatory-copy-propagation -canonical-ossa-rewrite-borrows -enable-sil-verify-all %s | %FileCheck %s --check-prefixes=CHECK,CHECK-ONONE +// RUN: %target-sil-opt -update-borrowed-from -copy-propagation -canonical-ossa-rewrite-borrows -enable-sil-verify-all %s | %FileCheck %s --check-prefixes=CHECK,CHECK-OPT +// RUN: %target-sil-opt -update-borrowed-from -mandatory-copy-propagation -canonical-ossa-rewrite-borrows -enable-sil-verify-all %s | %FileCheck %s --check-prefixes=CHECK,CHECK-ONONE // REQUIRES: asserts diff --git a/test/SILOptimizer/copy_propagation_borrow.sil b/test/SILOptimizer/copy_propagation_borrow.sil index 5f6ad04503f41..e2b087a947017 100644 --- a/test/SILOptimizer/copy_propagation_borrow.sil +++ b/test/SILOptimizer/copy_propagation_borrow.sil @@ -1,11 +1,12 @@ -// RUN: %target-sil-opt -copy-propagation -canonical-ossa-rewrite-borrows -enable-sil-verify-all -module-name Swift %s | %FileCheck %s -// RUN: %target-sil-opt -copy-propagation -enable-sil-verify-all -module-name Swift %s | %FileCheck %s --check-prefixes=CHECK-NOSCOPES +// RUN: %target-sil-opt -update-borrowed-from -copy-propagation -canonical-ossa-rewrite-borrows -enable-sil-verify-all -module-name Swift %s | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -copy-propagation -enable-sil-verify-all -module-name Swift %s | %FileCheck %s --check-prefixes=CHECK-NOSCOPES // // Most CopyPropagation tests are still in copy_propagation_opaque.sil. // // Mandatory copy propagation does not handles borrows yet. // REQUIRES: asserts +// REQUIRES: swift_in_compiler // arm64e checks that there is a sound Swift stdlib but the test compiles itself // as `-module Swift` and so that verification fails. (in some ptrauth @@ -565,8 +566,9 @@ bb0(%0 : @guaranteed $HasObject): // CHECK: br bb3([[CP]] : $C, [[BORROW]] : $C) // CHECK: bb2: // CHECK: br bb3([[CP]] : $C, [[BORROW]] : $C) -// CHECK: bb3([[OWNEDPHI:%.*]] : @owned $C, [[BORROWPHI:%.*]] @reborrow @guaranteed $C -// CHECK: end_borrow [[BORROWPHI]] +// CHECK: bb3([[OWNEDPHI:%.*]] : @owned $C, [[BORROWPHI:%.*]] : @reborrow @guaranteed $C): +// CHECK: [[R:%.*]] = borrowed [[BORROWPHI]] : $C from (%0 : $C) +// CHECK: end_borrow [[R]] // CHECK: destroy_value [[OWNEDPHI]] : $C // CHECK: destroy_value %0 : $C // CHECK-LABEL: } // end sil function 'testSubReborrowExtension' @@ -605,8 +607,9 @@ bb3(%phi : @owned $C, %borrowphi : @guaranteed $C): // CHECK-LABEL: sil [ossa] @testLiveCopyAfterReborrow : $@convention(thin) () -> () { // CHECK: [[ALLOC:%.*]] = alloc_ref $C // CHECK: bb3([[BORROWPHI:%.*]] : @reborrow @guaranteed $C): -// CHECK: [[COPY:%.*]] = copy_value [[BORROWPHI]] -// CHECK: end_borrow [[BORROWPHI]] : $C +// CHECK: [[R:%.*]] = borrowed [[BORROWPHI]] : $C from (%0 : $C) +// CHECK: [[COPY:%.*]] = copy_value [[R]] +// CHECK: end_borrow [[R]] : $C // CHECK-NOT: copy_value // CHECK-NOT: destroy_value // CHECK: apply @@ -653,8 +656,9 @@ bb3(%borrowphi : @guaranteed $C): // CHECK-LABEL: sil [ossa] @testDeadCopyAfterReborrow : $@convention(thin) () -> () { // CHECK: [[ALLOC:%.*]] = alloc_ref $C // CHECK: bb3([[BORROWPHI:%.*]] : @reborrow @guaranteed $C): +// CHECK: [[R:%.*]] = borrowed [[BORROWPHI]] : $C from (%0 : $C) // CHECK-NOT: copy_value -// CHECK: end_borrow [[BORROWPHI]] : $C +// CHECK: end_borrow [[R]] : $C // CHECK-NOT: copy_value // CHECK: destroy_value [[ALLOC]] : $C // CHECK-LABEL: } // end sil function 'testDeadCopyAfterReborrow' @@ -693,8 +697,9 @@ bb3(%borrowphi : @guaranteed $C): // CHECK-LABEL: sil [ossa] @testNestedReborrowOutsideUse : $@convention(thin) () -> () { // CHECK: [[ALLOC:%.*]] = alloc_ref $C // CHECK: bb3([[BORROWPHI:%.*]] : @reborrow @guaranteed $C): +// CHECK: [[R:%.*]] = borrowed [[BORROWPHI]] : $C from (%0 : $C) // CHECK-NOT: copy -// CHECK: end_borrow [[BORROWPHI]] +// CHECK: end_borrow [[R]] // CHECK-NEXT: destroy_value [[ALLOC]] : $C // CHECK-LABEL: } // end sil function 'testNestedReborrowOutsideUse' sil [ossa] @testNestedReborrowOutsideUse : $@convention(thin) () -> () { diff --git a/test/SILOptimizer/copy_propagation_canonicalize_with_reborrows.sil b/test/SILOptimizer/copy_propagation_canonicalize_with_reborrows.sil index 94ad5cf0f2792..8f7ef2c930ae5 100644 --- a/test/SILOptimizer/copy_propagation_canonicalize_with_reborrows.sil +++ b/test/SILOptimizer/copy_propagation_canonicalize_with_reborrows.sil @@ -1,4 +1,6 @@ -// RUN: %target-sil-opt -copy-propagation -enable-sil-verify-all %s | %FileCheck %s --check-prefixes=CHECK,CHECK-OPT +// RUN: %target-sil-opt -update-borrowed-from -copy-propagation -enable-sil-verify-all %s | %FileCheck %s --check-prefixes=CHECK,CHECK-OPT + +// REQUIRES: swift_in_compiler class X {} @@ -12,7 +14,8 @@ sil [ossa] @holdX : $@convention(thin) (@guaranteed X) -> () // CHECK-LABEL: sil [ossa] @nohoist_destroy_over_reborrow_endborrow : {{.*}} { // CHECK: {{bb[0-9]+}}([[REBORROW:%[^,]+]] : @reborrow @guaranteed $X, [[VALUE:%[^,]+]] : @owned $X): -// CHECK: end_borrow [[REBORROW]] +// CHECK: [[BF:%.*]] = borrowed [[REBORROW]] : $X from ([[VALUE]] : $X) +// CHECK: end_borrow [[BF]] // CHECK: destroy_value [[VALUE]] // CHECK-LABEL: } // end sil function 'nohoist_destroy_over_reborrow_endborrow' sil [ossa] @nohoist_destroy_over_reborrow_endborrow : $@convention(thin) () -> () { @@ -35,7 +38,8 @@ exit: // CHECK-LABEL: sil [ossa] @hoist_destroy_over_unrelated_endborrow : {{.*}} { // CHECK: {{bb[0-9]+}}([[UNRELATED:%[^,]+]] : @reborrow @guaranteed $X, [[VALUE:%[^,]+]] : @owned $X): // CHECK: destroy_value [[VALUE]] -// CHECK: end_borrow [[UNRELATED]] +// CHECK: [[BF:%.*]] = borrowed [[UNRELATED]] : $X from (%0 : $X) +// CHECK: end_borrow [[BF]] // CHECK-LABEL: } // end sil function 'hoist_destroy_over_unrelated_endborrow' sil [ossa] @hoist_destroy_over_unrelated_endborrow : $@convention(thin) (@guaranteed X) -> () { entry(%unrelated : @guaranteed $X): @@ -59,7 +63,8 @@ exit: // CHECK-LABEL: sil [ossa] @nohoist_destroy_over_reborrow_reborrow_endborrow : {{.*}} { // CHECK: {{bb[0-9]+}}({{%[^,]+}} : @reborrow @guaranteed $X, {{%[^,]+}} : @owned $X): // CHECK: {{bb[0-9]+}}([[REBORROW:%[^,]+]] : @reborrow @guaranteed $X, [[VALUE:%[^,]+]] : @owned $X): -// CHECK: end_borrow [[REBORROW]] +// CHECK: [[BF:%.*]] = borrowed [[REBORROW]] : $X from ([[VALUE]] : $X) +// CHECK: end_borrow [[BF]] // CHECK: destroy_value [[VALUE]] // CHECK-LABEL: } // end sil function 'nohoist_destroy_over_reborrow_reborrow_endborrow' sil [ossa] @nohoist_destroy_over_reborrow_reborrow_endborrow : $@convention(thin) () -> () { @@ -86,7 +91,8 @@ exit: // CHECK: {{bb[0-9]+}}({{%[^,]+}} : @reborrow @guaranteed $X, {{%[^,]+}} : @owned $X): // CHECK: {{bb[0-9]+}}([[REBORROW:%[^,]+]] : @reborrow @guaranteed $X, [[VALUE:%[^,]+]] : @owned $X): // CHECK: destroy_value [[VALUE]] -// CHECK: end_borrow [[REBORROW]] +// CHECK: [[BF:%.*]] = borrowed [[REBORROW]] : $X from (%0 : $X) +// CHECK: end_borrow [[BF]] // CHECK-LABEL: } // end sil function 'hoist_destroy_over_unrelated_reborrow_reborrow_endborrow' sil [ossa] @hoist_destroy_over_unrelated_reborrow_reborrow_endborrow : $@convention(thin) (@guaranteed X) -> () { entry(%unrelated : @guaranteed $X): @@ -112,7 +118,8 @@ exit: // CHECK-LABEL: sil [ossa] @nohoist_destroy_over_forward_reborrow_endborrow : {{.*}} { // CHECK: {{bb[0-9]+}}([[REBORROW:%[^,]+]] : @reborrow @guaranteed $X, [[VALUE:%[^,]+]] : @owned $X): -// CHECK: end_borrow [[REBORROW]] +// CHECK: [[BF:%.*]] = borrowed [[REBORROW]] : $X from ([[VALUE]] : $X) +// CHECK: end_borrow [[BF]] // CHECK: destroy_value [[VALUE]] // CHECK-LABEL: } // end sil function 'nohoist_destroy_over_forward_reborrow_endborrow' sil [ossa] @nohoist_destroy_over_forward_reborrow_endborrow : $@convention(thin) () -> () { @@ -137,12 +144,13 @@ exit: // CHECK-LABEL: sil [ossa] @nohoist_destroy_over_reborrow_loop : {{.*}} { // CHECK: {{bb[0-9]+}}([[REBORROW:%[^,]+]] : @reborrow @guaranteed $FakeOptional, [[VALUE:%[^,]+]] : @owned $FakeOptional): +// CHECK: [[BF:%.*]] = borrowed [[REBORROW]] : $FakeOptional from ([[VALUE]] : $FakeOptional) // CHECK: cond_br undef, [[LOOP:bb[0-9]+]], [[BODY:bb[0-9]+]] // CHECK: [[LOOP]]: -// CHECK: end_borrow [[REBORROW]] : $FakeOptional +// CHECK: end_borrow [[BF]] : $FakeOptional // CHECK: destroy_value [[VALUE]] : $FakeOptional // CHECK: [[BODY]]: -// CHECK: end_borrow [[REBORROW]] : $FakeOptional +// CHECK: end_borrow [[BF]] : $FakeOptional // CHECK: destroy_value [[VALUE]] : $FakeOptional // CHECK-LABEL: } // end sil function 'nohoist_destroy_over_reborrow_loop' sil [ossa] @nohoist_destroy_over_reborrow_loop : $@convention(thin) () -> () { @@ -176,8 +184,9 @@ exit: // the reborrow occurs. // CHECK-LABEL: sil [ossa] @reborrow_adjacent_to_consume_doesnt_extend_lifetime : {{.*}} { // CHECK: bb1([[REBORROW:%[^,]+]] : @reborrow @guaranteed $X, [[VALUE:%[^,]+]] : +// CHECK: [[R:%.*]] = borrowed [[REBORROW]] : $X from ([[VALUE]] : $X) // CHECK: {{bb[0-9]+}}: -// CHECK: end_borrow [[REBORROW]] +// CHECK: end_borrow [[R]] // CHECK: destroy_value [[VALUE]] // CHECK-LABEL: } // end sil function 'reborrow_adjacent_to_consume_doesnt_extend_lifetime' sil [ossa] @reborrow_adjacent_to_consume_doesnt_extend_lifetime : $@convention(thin) (@owned X) -> () { @@ -299,8 +308,10 @@ bb1(%4 : @guaranteed $X, %5 : @owned $X): // // CHECK-LABEL: sil [ossa] @reborrow_adjacent_to_consume : $@convention(thin) () -> () { // CHECK: bb1(%{{.*}} : @reborrow @guaranteed $X, %{{.*}} : @owned $X): +// CHECK-NEXT: borrowed // CHECK-NEXT: br bb2 // CHECK: bb2(%{{.*}} : @reborrow @guaranteed $X, %{{.*}} : @owned $X): +// CHECK-NEXT: borrowed // CHECK-NEXT: end_borrow // CHECK-NEXT: destroy_value sil [ossa] @reborrow_adjacent_to_consume : $@convention(thin) () -> () { @@ -331,8 +342,10 @@ bb2(%6 : @guaranteed $X, %7 : @owned $X): // // CHECK-LABEL: sil [ossa] @reborrow_no_adjacent_consume : $@convention(thin) () -> () { // CHECK: bb1(%{{.*}} : @reborrow @guaranteed $X, %{{.*}} : @owned $X): +// CHECK-NEXT: borrowed // CHECK-NEXT: br bb2 // CHECK: bb2(%{{.*}} : @reborrow @guaranteed $X): +// CHECK-NEXT: borrowed // CHECK-NEXT: end_borrow // CHECK-NEXT: destroy_value sil [ossa] @reborrow_no_adjacent_consume : $@convention(thin) () -> () { diff --git a/test/SILOptimizer/cse_ossa.sil b/test/SILOptimizer/cse_ossa.sil index 2733c247e7991..f9593fc547df4 100644 --- a/test/SILOptimizer/cse_ossa.sil +++ b/test/SILOptimizer/cse_ossa.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-sil-verify-all %s -cse | %FileCheck %s +// RUN: %target-sil-opt -enable-sil-verify-all %s -update-borrowed-from -cse | %FileCheck %s sil_stage canonical import Builtin diff --git a/test/SILOptimizer/cse_ossa_nontrivial.sil b/test/SILOptimizer/cse_ossa_nontrivial.sil index a6d14d6ffaa7c..0d8960bd8c2e8 100644 --- a/test/SILOptimizer/cse_ossa_nontrivial.sil +++ b/test/SILOptimizer/cse_ossa_nontrivial.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-sil-verify-all %s -cse | %FileCheck %s +// RUN: %target-sil-opt -enable-sil-verify-all %s -update-borrowed-from -cse | %FileCheck %s sil_stage canonical import Builtin diff --git a/test/SILOptimizer/dead_code_elimination_nontrivial_ossa.sil b/test/SILOptimizer/dead_code_elimination_nontrivial_ossa.sil index 6bd8a340bc183..89f879d26bb4b 100644 --- a/test/SILOptimizer/dead_code_elimination_nontrivial_ossa.sil +++ b/test/SILOptimizer/dead_code_elimination_nontrivial_ossa.sil @@ -1,4 +1,6 @@ -// RUN: %target-sil-opt -enable-sil-verify-all -dce %s | %FileCheck %s +// RUN: %target-sil-opt -enable-sil-verify-all -update-borrowed-from -dce %s | %FileCheck %s + +// REQUIRES: swift_in_compiler sil_stage canonical @@ -803,7 +805,8 @@ exit: // CHECK: [[OUTER_LIFETIME_1:%[^,]+]] = begin_borrow [[INSTANCE]] // CHECK: br [[EXIT:bb[0-9]+]]([[OUTER_LIFETIME_1]] : $Klass) // CHECK: [[EXIT]]([[OUTER_LIFETIME_2:%[^,]+]] : @reborrow @guaranteed $Klass): -// CHECK: end_borrow [[OUTER_LIFETIME_2]] +// CHECK: [[R:%.*]] = borrowed [[OUTER_LIFETIME_2]] : $Klass from (%0 : $Klass) +// CHECK: end_borrow [[R]] // CHECK: destroy_value [[INSTANCE]] // CHECK: [[RETVAL:%[^,]+]] = tuple () // CHECK: return [[RETVAL]] @@ -826,9 +829,11 @@ exit(%outer_lifetime_2 : @guaranteed $Klass, %inner_lifetime_2 : @guaranteed $Kl // CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] // CHECK: br [[WORK:bb[0-9]+]]([[LIFETIME]] : $Klass) // CHECK: [[WORK]]([[LIFETIME_1:%[^,]+]] : @reborrow @guaranteed $Klass): -// CHECK: br [[EXIT:bb[0-9]+]]([[LIFETIME_1]] : $Klass) +// CHECK: [[R:%.*]] = borrowed [[LIFETIME_1]] : $Klass from (%0 : $Klass) +// CHECK: br [[EXIT:bb[0-9]+]]([[R]] : $Klass) // CHECK: [[EXIT]]([[LIFETIME_2:%[^,]+]] : @reborrow @guaranteed $Klass): -// CHECK: end_borrow [[LIFETIME_2]] +// CHECK: [[R2:%.*]] = borrowed [[LIFETIME_2]] : $Klass from (%0 : $Klass) +// CHECK: end_borrow [[R2]] // CHECK: destroy_value [[INSTANCE]] // CHECK: [[RETVAL:%[^,]+]] = tuple () // CHECK: return [[RETVAL]] : $() @@ -908,7 +913,8 @@ exit(%inner_lifetime_2 : @guaranteed $Klass): // CHECK: [[LIFETIME:%[^,]+]] = load_borrow [[INSTANCE]] // CHECK: br [[BASIC_BLOCK1:bb[0-9]+]]([[LIFETIME]] : $Klass) // CHECK: [[BASIC_BLOCK1]]([[LIFETIME_2:%[^,]+]] : @reborrow @guaranteed $Klass): -// CHECK: end_borrow [[LIFETIME_2]] +// CHECK: [[R:%.*]] = borrowed [[LIFETIME_2]] : $Klass from () +// CHECK: end_borrow [[R]] // CHECK: [[RETVAL:%[^,]+]] = tuple () // CHECK: return [[RETVAL]] // CHECK-LABEL: } // end sil function 'reborrow_load_borrow' @@ -929,7 +935,8 @@ bb1(%4 : @guaranteed $Klass, %5 : @guaranteed $Klass): // CHECK: [[LIFETIME:%[^,]+]] = load_borrow [[ADDR]] : $*Klass // CHECK: br [[EXIT:bb[0-9]+]]([[LIFETIME]] : $Klass) // CHECK: [[EXIT]]([[LIFETIME_2:%[^,]+]] : @reborrow @guaranteed $Klass): -// CHECK: end_borrow [[LIFETIME_2]] : $Klass +// CHECK: [[R:%.*]] = borrowed [[LIFETIME_2]] : $Klass from () +// CHECK: end_borrow [[R]] : $Klass // CHECK: [[EXIT:%[^,]+]] = tuple () // CHECK: return [[EXIT]] : $() // CHECK-LABEL: } // end sil function 'reborrow_load_borrow2' diff --git a/test/SILOptimizer/enclosing_def_unit.sil b/test/SILOptimizer/enclosing_def_unit.sil index 0f0e700b4093d..d581dceafcebd 100644 --- a/test/SILOptimizer/enclosing_def_unit.sil +++ b/test/SILOptimizer/enclosing_def_unit.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -test-runner %s -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -test-runner -sil-disable-input-verify %s -o /dev/null 2>&1 | %FileCheck %s // // REQUIRES: swift_in_compiler @@ -121,7 +121,7 @@ exit: // CHECK-LABEL: enclosing_def_single_introducer: enclosing_values with: %aggregate2 // CHECK: sil [ossa] @enclosing_def_single_introducer -// CHECK: Enclosing values for: %{{.*}} = struct $PairC (%10 : $C, %11 : $C) +// CHECK: Enclosing values for: %{{.*}} = struct $PairC (%11 : $C, %12 : $C) // CHECK: begin_borrow %0 : $C // CHECK-NEXT: enclosing_def_single_introducer: enclosing_values with: %aggregate2 sil [ossa] @enclosing_def_single_introducer : $@convention(thin) (@guaranteed C) -> () { @@ -265,8 +265,8 @@ exit: // CHECK-LABEL: enclosing_def_example: enclosing_values with: %first // CHECK: Enclosing values for: %{{.*}} = struct_extract %{{.*}} : $PairC, #PairC.first -// CHECK-NEXT: %{{.*}} = begin_borrow %0 : $C // CHECK-NEXT: %{{.*}} = argument of bb0 : $C +// CHECK-NEXT: %{{.*}} = begin_borrow %0 : $C // CHECK-NEXT: enclosing_def_example: enclosing_values with: %first // CHECK-LABEL: enclosing_def_example: enclosing_values with: %field diff --git a/test/SILOptimizer/field_sensitive_liverange_unit.sil b/test/SILOptimizer/field_sensitive_liverange_unit.sil index ea2136a667ab0..53dcfb0655d98 100644 --- a/test/SILOptimizer/field_sensitive_liverange_unit.sil +++ b/test/SILOptimizer/field_sensitive_liverange_unit.sil @@ -12,7 +12,7 @@ sil @getC : $@convention(thin) () -> (@owned C) // CHECK-LABEL: testReborrow: fieldsensitive-multidefuse-liverange // CHECK: FieldSensitive MultiDef lifetime analysis: // CHECK: def in range [0, 1) value: [[B:%.*]] = load_borrow %0 : $*C -// CHECK: def in range [0, 1) value: [[RB:%.*]] = argument of bb3 : $C +// CHECK: def in range [0, 1) value: [[RB:%.*]] = borrowed {{.*}} from // CHECK-NEXT: bb2: LiveWithin // CHECK-NEXT: bb3: LiveWithin // All users are printed here. @@ -30,7 +30,7 @@ bb0: @trace[1] 0 1 uses: @block[2].instruction[3] true 0 1 - @block[3].instruction[0] true 0 1 + @block[3].instruction[1] true 0 1 """ %stack = alloc_stack $C %getC = function_ref @getC : $@convention(thin) () -> (@owned C) @@ -49,7 +49,8 @@ bb2: debug_value [trace] %borrow2 : $C br bb3(%borrow2 : $C) -bb3(%reborrow : @guaranteed $C): +bb3(%a : @guaranteed $C): + %reborrow = borrowed %a : $C from () debug_value [trace] %reborrow : $C end_borrow %reborrow : $C br bb4 diff --git a/test/SILOptimizer/let-property-lowering.sil b/test/SILOptimizer/let-property-lowering.sil index 6627cc924afe2..bb4b318df571e 100644 --- a/test/SILOptimizer/let-property-lowering.sil +++ b/test/SILOptimizer/let-property-lowering.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt %s -let-property-lowering | %FileCheck %s +// RUN: %target-sil-opt %s -update-borrowed-from -let-property-lowering | %FileCheck %s // REQUIRES: swift_in_compiler @@ -291,7 +291,8 @@ bb2: // CHECK-LABEL: sil [ossa] @test_reborrow : // CHECK: bb1([[A:%.*]] : @reborrow @guaranteed $C): -// CHECK: end_borrow [[A]] +// CHECK: [[R:%.*]] = borrowed [[A]] : $C from (%2 : $C) +// CHECK: end_borrow [[R]] // CHECK-NEXT: end_init_let_ref %2 // CHECK: } // end sil function 'test_reborrow' sil [ossa] @test_reborrow : $@convention(thin) (@owned C, Int) -> @owned C { diff --git a/test/SILOptimizer/lexical_destroy_hoisting.sil b/test/SILOptimizer/lexical_destroy_hoisting.sil index a1f89f242269a..0a1138e0515cd 100644 --- a/test/SILOptimizer/lexical_destroy_hoisting.sil +++ b/test/SILOptimizer/lexical_destroy_hoisting.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -compute-side-effects -copy-propagation -enable-sil-verify-all %s | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -compute-side-effects -copy-propagation -enable-sil-verify-all %s | %FileCheck %s // REQUIRES: swift_in_compiler @@ -187,7 +187,8 @@ exit: // CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] // CHECK: br [[WORK:bb[0-9]+]]([[LIFETIME]] : $C) // CHECK: [[WORK]]([[LIFETIME_2:%[^,]+]] : @reborrow @guaranteed $C): -// CHECK: end_borrow [[LIFETIME_2]] +// CHECK: [[R:%.*]] = borrowed [[LIFETIME_2]] : $C from (%0 : $C) +// CHECK: end_borrow [[R]] // CHECK: tuple () // CHECK: destroy_value [[INSTANCE]] // CHECK: br diff --git a/test/SILOptimizer/liveness_unit.sil b/test/SILOptimizer/liveness_unit.sil index 3baa4c7bda6dc..b8cdca8bd2db9 100644 --- a/test/SILOptimizer/liveness_unit.sil +++ b/test/SILOptimizer/liveness_unit.sil @@ -1,4 +1,6 @@ -// RUN: %target-sil-opt -test-runner %s -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -test-runner %s -o /dev/null 2>&1 | %FileCheck %s + +// REQUIRES: swift_in_compiler sil_stage canonical @@ -80,7 +82,7 @@ bb3: // CHECK-LABEL: testReborrow: multidef_liveness // CHECK: MultiDef lifetime analysis: // CHECK: def value: [[B:%.*]] = begin_borrow %0 : $C -// CHECK: def value: [[RB:%.*]] = argument of bb3 : $C +// CHECK: def value: [[RB:%.*]] = borrowed {{.*}} from (%0 : $C) // CHECK-NEXT: bb2: LiveWithin // CHECK-NEXT: bb3: LiveWithin // CHECK-NEXT: lifetime-ending user: br bb3([[B]] : $C) @@ -225,10 +227,10 @@ bb4(%phi : @owned $C): // CHECK-LABEL: testSSADeadRefElementAddr: ssa_liveness // CHECK: SSA lifetime analysis: %0 = argument of bb0 : $C // CHECK-NEXT: bb0: LiveOut -// CHECK-NEXT: bb2: LiveOut // CHECK-NEXT: bb3: LiveWithin +// CHECK-NEXT: bb2: LiveOut // CHECK-NEXT: bb1: LiveOut -// CHECK: ref_element_addr %6 : $D, #D.object +// CHECK: ref_element_addr %7 : $D, #D.object // CHECK-NEXT-LABEL: end running test 1 of 1 on testSSADeadRefElementAddr: ssa_liveness sil [ossa] @testSSADeadRefElementAddr : $@convention(thin) (@guaranteed C) -> () { bb0(%0 : @guaranteed $C): @@ -254,9 +256,11 @@ bb3(%phi : @guaranteed $D): // CHECK-LABEL: testSSAInnerReborrowedPhi: ssa_liveness // CHECK: SSA lifetime analysis: %0 = argument of bb0 : $C // CHECK-NEXT: Incomplete liveness: Reborrowed inner scope -// CHECK-NEXT: bb0: LiveWithin -// CHECK-NEXT: regular user: br bb1( -// CHECK-NEXT: last user: br bb1( +// CHECK-NEXT: bb0: LiveOut +// CHECK-NEXT: bb1: LiveWithin +// CHECK-NEXT: regular user: %7 = borrowed %4 : $C from (%0 : $C) +// CHECK-NEXT: regular user: br bb1(%1 : $C, %2 : $PairC) +// CHECK-NEXT: last user: %7 = borrowed %4 : $C from (%0 : $C) // CHECK-NEXT: end running sil [ossa] @testSSAInnerReborrowedPhi : $@convention(thin) (@guaranteed C) -> () { bb0(%0 : @guaranteed $C): @@ -285,12 +289,11 @@ bb1(%reborrow : @guaranteed $C, %phi : @guaranteed $PairC): // CHECK-LABEL: testSSADeadGuaranteedPhi: ssa_liveness // CHECK: SSA lifetime analysis: %0 = argument of bb0 : $C // CHECK-NEXT: bb0: LiveOut -// CHECK-NEXT: bb2: LiveWithin -// CHECK-NEXT: bb1: LiveWithin +// CHECK-NEXT: bb3: LiveWithin +// CHECK-NEXT: bb2: LiveOut +// CHECK-NEXT: bb1: LiveOut // CHECK-NEXT: regular user -// CHECK: last user: -// CHECK-SAME: br bb3 -// CHECK-NEXT: last user: br bb3 +// CHECK: last user: %7 = borrowed %6 : $D from (%0 : $C) // CHECK-NEXT: end running sil [ossa] @testSSADeadGuaranteedPhi : $@convention(thin) (@guaranteed C) -> () { bb0(%0 : @guaranteed $C): @@ -315,9 +318,10 @@ bb3(%deadPhi : @guaranteed $D): // CHECK-LABEL: testSSADominatedPhi: ssa_liveness // CHECK: SSA lifetime analysis: %0 = argument of bb0 : $C // CHECK-NEXT: bb0: LiveOut -// CHECK-NEXT: bb2: LiveOut // CHECK-NEXT: bb3: LiveWithin +// CHECK-NEXT: bb2: LiveOut // CHECK-NEXT: bb1: LiveOut +// CHECK-NEXT: regular user: %{{.*}} = borrowed %6 : $D from (%0 : $C) // CHECK-NEXT: regular user: %{{.*}} = unchecked_ref_cast %0 : $C to $D // CHECK-NEXT: regular user: br bb3 // CHECK-NEXT: regular user: %{{.*}} = load diff --git a/test/SILOptimizer/loop_unroll_ossa.sil b/test/SILOptimizer/loop_unroll_ossa.sil index 2fe6f6e628a06..40e6cb71c8d32 100644 --- a/test/SILOptimizer/loop_unroll_ossa.sil +++ b/test/SILOptimizer/loop_unroll_ossa.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-sil-verify-all -loop-unroll %s | %FileCheck %s +// RUN: %target-sil-opt -enable-sil-verify-all -update-borrowed-from -loop-unroll %s | %FileCheck %s sil_stage canonical diff --git a/test/SILOptimizer/looprotate_nontrivial_ossa.sil b/test/SILOptimizer/looprotate_nontrivial_ossa.sil index b92c58d0ddc52..4ff9e7e03fda7 100644 --- a/test/SILOptimizer/looprotate_nontrivial_ossa.sil +++ b/test/SILOptimizer/looprotate_nontrivial_ossa.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -loop-rotate -looprotate-single-block-loop=true %s | %FileCheck %s +// RUN: %target-sil-opt -loop-rotate -update-borrowed-from -looprotate-single-block-loop=true %s | %FileCheck %s sil_stage canonical import Builtin diff --git a/test/SILOptimizer/ossa_rauw_tests.sil b/test/SILOptimizer/ossa_rauw_tests.sil index e6ef0b4282ad7..89764eadf29b9 100644 --- a/test/SILOptimizer/ossa_rauw_tests.sil +++ b/test/SILOptimizer/ossa_rauw_tests.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-sil-verify-all -sil-combine -semantic-arc-opts %s | %FileCheck %s +// RUN: %target-sil-opt -enable-sil-verify-all -update-borrowed-from -sil-combine -semantic-arc-opts %s | %FileCheck %s // Make sure that we can perform all of these RAUW without producing ARC traffic // that semantic arc opts can't eliminate. diff --git a/test/SILOptimizer/ownership_liveness_unit.sil b/test/SILOptimizer/ownership_liveness_unit.sil index 2d40151ea775a..290f725a7786f 100644 --- a/test/SILOptimizer/ownership_liveness_unit.sil +++ b/test/SILOptimizer/ownership_liveness_unit.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -test-runner %s -o /dev/null 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -test-runner %s -o /dev/null 2>&1 | %FileCheck %s // // TODO: when complete lifetime verification become the default for SIL verification, // then consider moving all tests with 'unreachable' into a separate file with the flag @@ -425,14 +425,14 @@ bb3(%reborrow1 : @guaranteed $C, %reborrow2 : @guaranteed $D): // CHECK-NEXT: Inner scope: %{{.*}} = argument of bb1 : $D // CHECK-NEXT: bb0: LiveOut // CHECK-NEXT: bb1: LiveWithin +// CHECK-NEXT: regular user: %{{.*}} = borrowed %{{.*}} : $D from (%1 : $C) // CHECK-NEXT: lifetime-ending user: end_borrow %{{.*}} : $C // CHECK-NEXT: regular user: %{{.*}} = unchecked_ref_cast %{{.*}} : $C to $D -// CHECK-NEXT: regular user: end_borrow %{{.*}} : $D // CHECK-NEXT: regular user: br bb1(%{{.*}} : $D) // CHECK-NEXT: Complete liveness // CHECK-NEXT: Unenclosed phis { // CHECK-NEXT: } -// CHECK-NEXT: last user: end_borrow %1 : $C // id: %7 +// CHECK-NEXT: last user: end_borrow %1 : $C // CHECK-NEXT: testInteriorDominatedReborrow: interior-liveness with: %borrow1 // CHECK-LABEL: testInteriorDominatedReborrow: interior_liveness_swift with: %borrow1 @@ -441,6 +441,7 @@ bb3(%reborrow1 : @guaranteed $C, %reborrow2 : @guaranteed $D): // CHECK-NEXT: ends: end_borrow %{{.*}} : $C // CHECK-NEXT: exits: // CHECK-NEXT: interiors: end_borrow %{{.*}} : $D +// CHECK-NEXT: %{{.*}} = borrowed %{{.*}} : $D from (%1 : $C) // CHECK-NEXT: br bb1(%{{.*}} : $D) // CHECK-NEXT: %{{.*}} = unchecked_ref_cast %{{.*}} : $C to $D // CHECK-NEXT: Unenclosed phis { @@ -588,9 +589,10 @@ bb0(%0 : @guaranteed $D, %1 : @owned $C): // CHECK-NEXT: begin: cond_br undef, bb1, bb2 // CHECK-NEXT: ends: %{{.*}} = load [copy] %{{.*}} : $*C // CHECK-NEXT: exits: -// CHECK-NEXT: interiors: br bb3(%{{.*}} : $D) +// CHECK-NEXT: interiors: %{{.*}} = ref_element_addr %{{.*}} : $D, #D.object +// CHECK-NEXT: %{{.*}} = borrowed %{{.*}} : $D from (%0 : $C) +// CHECK-NEXT: br bb3(%{{.*}} : $D) // CHECK-NEXT: %{{.*}} = unchecked_ref_cast %0 : $C to $D -// CHECK-NEXT: %{{.*}} = ref_element_addr %{{.*}} : $D, #D.object // CHECK-NEXT: br bb3(%{{.*}} : $D) // CHECK-NEXT: %{{.*}} = unchecked_ref_cast %0 : $C to $D // CHECK-NEXT: Unenclosed phis { @@ -667,7 +669,7 @@ bb3(%reborrow : @guaranteed $C, %phi : @guaranteed $D): // CHECK: Complete liveness // CHECK: Unenclosed phis { // CHECK-NEXT: } -// CHECK-NEXT: last user: end_borrow +// CHECK-NEXT: last user: %{{.*}} = borrowed %{{.*}} : $C from (%0 : $C) // CHECK-NEXT: testInnerDominatedReborrow: interior-liveness with: @argument[0] // CHECK-LABEL: testInnerDominatedReborrow: interior_liveness_swift with: @argument[0] @@ -675,7 +677,8 @@ bb3(%reborrow : @guaranteed $C, %phi : @guaranteed $D): // CHECK-NEXT: begin: %{{.*}} = begin_borrow %0 : $C // CHECK-NEXT: ends: end_borrow %{{.*}} : $C // CHECK-NEXT: exits: -// CHECK-NEXT: interiors: br bb1(%{{.*}} : $C) +// CHECK-NEXT: interiors: %{{.*}} = borrowed %{{.*}} : $C from (%0 : $C) +// CHECK-NEXT: br bb1(%{{.*}} : $C) // CHECK-NEXT: Unenclosed phis { // CHECK-NEXT: } // CHECK-NEXT: last user: end_borrow %{{.*}} : $C @@ -708,6 +711,7 @@ bb1(%reborrow : @guaranteed $C): // CHECK-NEXT: ends: destroy_value %{{.*}} : $C // CHECK-NEXT: exits: // CHECK-NEXT: interiors: end_borrow %{{.*}} : $C +// CHECK-NEXT: %{{.*}} = borrowed %{{.*}} : $C from (%2 : $C, %1 : $C) // CHECK-NEXT: br bb3(%{{.*}} : $C) // CHECK-NEXT: Unenclosed phis { // CHECK-NEXT: } @@ -785,20 +789,23 @@ bb3(%outer : @guaranteed $D, %inner : @guaranteed $D): // CHECK: Interior liveness: [[DEF:%.*]] = argument of bb3 : $D // CHECK: Inner scope: %{{.*}} = argument of bb3 : $D // CHECK: Inner scope: %{{.*}} = argument of bb4 : $D -// CHECK: regular user: end_borrow -// CHECK-NEXT: regular user: br bb4( +// CHECK: regular user: br bb4( +// CHECK-NEXT: regular user: %{{.*}} = borrowed %{{.*}} : $D from +// CHECK-NEXT: regular user: %{{.*}} = borrowed %{{.*}} : $D from // CHECK-NEXT: Complete liveness // CHECK-NEXT: Unenclosed phis { // CHECK-NEXT: } -// CHECK-NEXT: last user: end_borrow +// CHECK-NEXT: last user: %{{.*}} = borrowed %{{.*}} : $D from // CHECK-NEXT: testInnerAdjacentReborrow1: interior-liveness with: %outer3 // CHECK-LABEL: testInnerAdjacentReborrow1: interior_liveness_swift with: %outer3 // CHECK: Interior liveness: %{{.*}} = argument of bb3 : $D -// CHECK-NEXT: begin: br bb4( +// CHECK-NEXT: begin: %{{.*}} = borrowed %{{.*}} : $D from // CHECK-NEXT: ends: end_borrow // CHECK-NEXT: exits: -// CHECK-NEXT: interiors: br bb4( +// CHECK-NEXT: interiors: %{{.*}} = borrowed %{{.*}} : $D from +// CHECK-NEXT: br bb4( +// CHECK-NEXT: %{{.*}} = borrowed %{{.*}} : $D from // CHECK-NEXT: Unenclosed phis { // CHECK-NEXT: } // CHECK-NEXT: last user: end_borrow @@ -834,20 +841,23 @@ bb4(%inner4 : @guaranteed $D): // CHECK: Interior liveness: [[DEF:%.*]] = argument of bb3 : $D // CHECK: Inner scope: %{{.*}} = argument of bb3 : $D // CHECK: Inner scope: %{{.*}} = argument of bb4 : $D -// CHECK: regular user: end_borrow -// CHECK-NEXT: regular user: br bb4( +// CHECK: regular user: br bb4( +// CHECK-NEXT: regular user: {{.*}} borrowed {{.*}} from +// CHECK-NEXT: regular user: {{.*}} borrowed {{.*}} from // CHECK-NEXT: Complete liveness // CHECK-NEXT: Unenclosed phis { // CHECK-NEXT: } -// CHECK-NEXT: last user: end_borrow +// CHECK-NEXT: last user: {{.*}} borrowed {{.*}} from // CHECK-NEXT: testInnerAdjacentReborrow2: interior-liveness with: %outer3 // CHECK-LABEL: testInnerAdjacentReborrow2: interior_liveness_swift with: %outer3 // CHECK: Interior liveness: %{{.*}} = argument of bb3 : $D -// CHECK-NEXT: begin: br bb4(%{{.*}} : $D) +// CHECK-NEXT: begin: {{.*}} borrowed {{.*}} from // CHECK-NEXT: ends: end_borrow %{{.*}} : $D // CHECK-NEXT: exits: -// CHECK-NEXT: interiors: br bb4(%{{.*}} : $D) +// CHECK-NEXT: interiors: {{.*}} borrowed {{.*}} from +// CHECK-NEXT: br bb4(%{{.*}} : $D) +// CHECK-NEXT: {{.*}} borrowed {{.*}} from // CHECK-NEXT: Unenclosed phis { // CHECK-NEXT: } // CHECK-NEXT: last user: end_borrow %{{.*}} : $D @@ -891,7 +901,7 @@ bb4(%inner4 : @guaranteed $D): // CHECK-LABEL: testInnerNonAdjacentReborrow: interior_liveness_swift with: %outer3 // CHECK: Interior liveness: %{{.*}} = argument of bb3 : $D -// CHECK-NEXT: begin: br bb4(%{{.*}} : $D) +// CHECK-NEXT: begin: {{.*}} borrowed {{.*}} from // CHECK-NEXT: ends: // CHECK-NEXT: exits: // CHECK-NEXT: interiors: @@ -936,11 +946,14 @@ bb4(%inner4 : @guaranteed $D): // CHECK-LABEL: testInnerAdjacentPhi1: interior_liveness_swift with: %inner3 // CHECK: Interior liveness: %{{.*}} = argument of bb3 : $C -// CHECK-NEXT: begin: br bb4(%{{.*}} : $D) +// CHECK-NEXT: begin: {{.*}} borrowed {{.*}} from // CHECK-NEXT: ends: %{{.*}} = load [copy] %{{.*}} : $*C // CHECK-NEXT: exits: -// CHECK-NEXT: interiors: br bb4(%{{.*}} : $D) -// CHECK-NEXT: %{{.*}} = ref_element_addr %{{.*}} : $D, #D.object +// CHECK-NEXT: interiors: %{{.*}} = ref_element_addr %{{.*}} : $D, #D.object +// CHECK-NEXT: {{.*}} borrowed {{.*}} from +// CHECK-NEXT: br bb4(%{{.*}} : $D) +// CHECK-NEXT: {{.*}} borrowed {{.*}} from +// CHECK-NEXT: {{.*}} borrowed {{.*}} from // CHECK-NEXT: Unenclosed phis { // CHECK-NEXT: } // CHECK-NEXT: last user: %{{.*}} = load [copy] %{{.*}} : $*C @@ -983,11 +996,14 @@ bb4(%phi4 : @guaranteed $D): // CHECK-LABEL: testInnerAdjacentPhi2: interior_liveness_swift with: %inner3 // CHECK: Interior liveness: %{{.*}} = argument of bb3 : $C -// CHECK-NEXT: begin: br bb4(%{{.*}} : $D) +// CHECK-NEXT: begin: {{.*}} borrowed {{.*}} from // CHECK-NEXT: ends: %{{.*}} = load [copy] %{{.*}} : $*C // CHECK-NEXT: exits: -// CHECK-NEXT: interiors: br bb4(%{{.*}} : $D) -// CHECK-NEXT: %{{.*}} = ref_element_addr %{{.*}} : $D, #D.object +// CHECK-NEXT: interiors: %{{.*}} = ref_element_addr %{{.*}} : $D, #D.object +// CHECK-NEXT: {{.*}} borrowed {{.*}} from +// CHECK-NEXT: br bb4(%{{.*}} : $D) +// CHECK-NEXT: {{.*}} borrowed {{.*}} from +// CHECK-NEXT: {{.*}} borrowed {{.*}} from // CHECK-NEXT: Unenclosed phis { // CHECK-NEXT: } // CHECK-NEXT: last user: %{{.*}} = load [copy] %{{.*}} : $*C @@ -1025,18 +1041,18 @@ bb4(%phi4 : @guaranteed $D): // CHECK: Complete liveness // CHECK: Unenclosed phis { // CHECK-NEXT: } -// CHECK-NEXT: dead def: [[DEF]] = argument of bb3 : $C +// CHECK-NEXT: last user: {{.*}} borrowed {{.*}} from // CHECK-NEXT: testInnerNonAdjacentPhi: interior-liveness with: %inner3 // CHECK-LABEL: testInnerNonAdjacentPhi: interior_liveness_swift with: %inner3 // CHECK: Interior liveness: %{{.*}} = argument of bb3 : $C -// CHECK-NEXT: begin: br bb4(%{{.*}} : $D) +// CHECK-NEXT: begin: {{.*}} borrowed {{.*}} from // CHECK-NEXT: ends: // CHECK-NEXT: exits: // CHECK-NEXT: interiors: // CHECK-NEXT: Unenclosed phis { // CHECK-NEXT: } -// CHECK-NEXT: dead def: %{{.*}} = argument of bb3 : $C +// CHECK-NEXT: last user: {{.*}} borrowed {{.*}} from // CHECK-NEXT: testInnerNonAdjacentPhi: interior_liveness_swift with: %inner3 sil [ossa] @testInnerNonAdjacentPhi : $@convention(thin) (@guaranteed C) -> () { bb0(%0 : @guaranteed $C): @@ -1097,90 +1113,6 @@ bb0(%0 : @guaranteed $D): return %99 : $() } -// CHECK-LABEL: testUnenclosedReborrow: interior-liveness with: %copy0a -// CHECK: Interior liveness: [[DEF:%.*]] = copy_value %0 : $C -// CHECK-NEXT: Inner scope: [[BORROW:%.*]] = begin_borrow [[DEF]] : $C -// CHECK: Complete liveness -// CHECK: Unenclosed phis { -// CHECK-NEXT: %{{.*}} = argument of bb3 : $C -// CHECK-NEXT: } -// CHECK-NEXT: last user: br bb3([[BORROW]] : $C) -// CHECK-NEXT: testUnenclosedReborrow: interior-liveness with: %copy0a - -// CHECK-LABEL: testUnenclosedReborrow: interior_liveness_swift with: %copy0a -// CHECK: Interior liveness: %{{.*}} = copy_value %0 : $C -// CHECK-NEXT: begin: %{{.*}} = copy_value %0 : $C -// CHECK-NEXT: ends: br bb3(%{{.*}} : $C) -// CHECK-NEXT: exits: -// CHECK-NEXT: interiors: -// CHECK-NEXT: Unenclosed phis { -// CHECK-NEXT: Phi(value: %{{.*}} = argument of bb3 : $C -// CHECK-NEXT: } -// CHECK-NEXT: last user: br bb3(%{{.*}} : $C) -// CHECK-NEXT: testUnenclosedReborrow: interior_liveness_swift with: %copy0a -sil [ossa] @testUnenclosedReborrow : $@convention(thin) (@guaranteed C) -> () { -bb0(%0 : @guaranteed $C): - cond_br undef, bb1, bb2 - -bb1: - %copy0a = copy_value %0 : $C - specify_test "interior-liveness %copy0a" - specify_test "interior_liveness_swift %copy0a" - %borrow1 = begin_borrow %copy0a : $C - br bb3(%borrow1 : $C) - -bb2: - %copy0b = copy_value %0 : $C - %borrow2 = begin_borrow %copy0b : $C - br bb3(%borrow2 : $C) - -bb3(%reborrow : @guaranteed $C): - end_borrow %reborrow : $C - unreachable -} - -// CHECK-LABEL: testUnenclosedGuaranteedPhi: interior-liveness with: %borrow1 -// CHECK: Interior liveness: %{{.*}} = begin_borrow -// CHECK: Complete liveness -// CHECK: Unenclosed phis { -// CHECK-NEXT: %{{.*}} = argument of bb3 : $D -// CHECK-NEXT: } -// CHECK-NEXT: last user: br bb3( -// CHECK-NEXT: testUnenclosedGuaranteedPhi: interior-liveness with: %borrow1 - -// CHECK-LABEL: testUnenclosedGuaranteedPhi: interior_liveness_swift with: %borrow1 -// CHECK: Interior liveness: %{{.*}} = begin_borrow %{{.*}} : $C -// CHECK-NEXT: begin: %{{.*}} = begin_borrow %{{.*}} : $C -// CHECK-NEXT: ends: br bb3(%{{.*}} : $D) -// CHECK-NEXT: exits: -// CHECK-NEXT: interiors: %{{.*}} = unchecked_ref_cast %{{.*}} : $C to $D -// CHECK-NEXT: Unenclosed phis { -// CHECK-NEXT: Phi(value: %{{.*}} = argument of bb3 : $D) -// CHECK-NEXT: } -// CHECK-NEXT: last user: br bb3(%{{.*}} : $D) -// CHECK-NEXT: testUnenclosedGuaranteedPhi: interior_liveness_swift with: %borrow1 -sil [ossa] @testUnenclosedGuaranteedPhi : $@convention(thin) (@guaranteed C) -> () { -bb0(%0 : @guaranteed $C): - cond_br undef, bb1, bb2 - -bb1: - %copy0a = copy_value %0 : $C - %borrow1 = begin_borrow %copy0a : $C - specify_test "interior-liveness %borrow1" - specify_test "interior_liveness_swift %borrow1" - %d1 = unchecked_ref_cast %borrow1 : $C to $D - br bb3(%d1 : $D) - -bb2: - %copy0b = copy_value %0 : $C - %borrow2 = begin_borrow %copy0b : $C - %d2 = unchecked_ref_cast %borrow2 : $C to $D - br bb3(%d2 : $D) - -bb3(%phi : @guaranteed $D): - unreachable -} - // ============================================================================= // ExtendedLiveness // ============================================================================= diff --git a/test/SILOptimizer/predictable_memopt_ownership.sil b/test/SILOptimizer/predictable_memopt_ownership.sil index 7321929d7f43c..afeb451e49a89 100644 --- a/test/SILOptimizer/predictable_memopt_ownership.sil +++ b/test/SILOptimizer/predictable_memopt_ownership.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-sil-verify-all %s -predictable-memaccess-opts -predictable-deadalloc-elim | %FileCheck %s +// RUN: %target-sil-opt -enable-sil-verify-all %s -update-borrowed-from -predictable-memaccess-opts -predictable-deadalloc-elim | %FileCheck %s sil_stage raw diff --git a/test/SILOptimizer/redundant_phi_elimination_ossa.sil b/test/SILOptimizer/redundant_phi_elimination_ossa.sil index 2629bb29ea9c9..c614216409741 100644 --- a/test/SILOptimizer/redundant_phi_elimination_ossa.sil +++ b/test/SILOptimizer/redundant_phi_elimination_ossa.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-sil-verify-all %s -redundant-phi-elimination | %FileCheck %s +// RUN: %target-sil-opt -enable-sil-verify-all %s -update-borrowed-from -redundant-phi-elimination | %FileCheck %s sil_stage canonical diff --git a/test/SILOptimizer/semantic-arc-opt-owned-to-guaranteed-phi.sil b/test/SILOptimizer/semantic-arc-opt-owned-to-guaranteed-phi.sil index ddda9dd429ad5..4b0fd9f24272d 100644 --- a/test/SILOptimizer/semantic-arc-opt-owned-to-guaranteed-phi.sil +++ b/test/SILOptimizer/semantic-arc-opt-owned-to-guaranteed-phi.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -semantic-arc-opts %s | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -semantic-arc-opts %s | %FileCheck %s class Klass {} diff --git a/test/SILOptimizer/semantic-arc-opts-canonical.sil b/test/SILOptimizer/semantic-arc-opts-canonical.sil index 8861b1c02f4ff..9310b6f740961 100644 --- a/test/SILOptimizer/semantic-arc-opts-canonical.sil +++ b/test/SILOptimizer/semantic-arc-opts-canonical.sil @@ -1,5 +1,7 @@ // RUN: %target-sil-opt -module-name Swift -enable-sil-verify-all -semantic-arc-opts %s | %FileCheck %s +// REQUIRES: swift_in_compiler + sil_stage canonical import Builtin @@ -462,7 +464,8 @@ bb3: // CHECK-NEXT: br bb3([[NONE]] : $FakeOptional) // // CHECK: bb3([[PHI:%.*]] : @guaranteed $FakeOptional): -// CHECK: switch_enum [[PHI]] : $FakeOptional, case #FakeOptional.some!enumelt: bb5, case #FakeOptional.none!enumelt: bb4 +// CHECK: [[BF:%.*]] = borrowed [[PHI]] +// CHECK: switch_enum [[BF]] : $FakeOptional, case #FakeOptional.some!enumelt: bb5, case #FakeOptional.none!enumelt: bb4 // // CHECK: bb4: // CHECK-NEXT: br bb6 diff --git a/test/SILOptimizer/semantic-arc-opts-loadcopy-to-loadborrow.sil b/test/SILOptimizer/semantic-arc-opts-loadcopy-to-loadborrow.sil index 188d682e0c7da..83874092379d4 100644 --- a/test/SILOptimizer/semantic-arc-opts-loadcopy-to-loadborrow.sil +++ b/test/SILOptimizer/semantic-arc-opts-loadcopy-to-loadborrow.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -module-name Swift -enable-sil-verify-all -semantic-arc-opts -sil-semantic-arc-peepholes-loadcopy-to-loadborrow -sil-semantic-arc-peepholes-redundant-borrowscope-elim %s | %FileCheck %s +// RUN: %target-sil-opt -module-name Swift -enable-sil-verify-all -update-borrowed-from -semantic-arc-opts -sil-semantic-arc-peepholes-loadcopy-to-loadborrow -sil-semantic-arc-peepholes-redundant-borrowscope-elim %s | %FileCheck %s // NOTE: After we run load [copy] -> load_borrow, we run one additional round of // borrow scope elimination since borrow scope elimination cleans up the IR a diff --git a/test/SILOptimizer/semantic-arc-opts.sil b/test/SILOptimizer/semantic-arc-opts.sil index f25e35d24a5e6..df5d14dcce0b3 100644 --- a/test/SILOptimizer/semantic-arc-opts.sil +++ b/test/SILOptimizer/semantic-arc-opts.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -module-name Swift -enable-sil-verify-all -semantic-arc-opts %s | %FileCheck %s +// RUN: %target-sil-opt -module-name Swift -enable-sil-verify-all -update-borrowed-from -semantic-arc-opts %s | %FileCheck %s sil_stage raw diff --git a/test/SILOptimizer/shrink_borrow_scope.sil b/test/SILOptimizer/shrink_borrow_scope.sil index 2100ae5d67587..fc689d8da959a 100644 --- a/test/SILOptimizer/shrink_borrow_scope.sil +++ b/test/SILOptimizer/shrink_borrow_scope.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -compute-side-effects -copy-propagation -enable-sil-verify-all %s | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -compute-side-effects -copy-propagation -enable-sil-verify-all %s | %FileCheck %s // REQUIRES: swift_in_compiler @@ -360,12 +360,13 @@ exit: // CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] // CHECK: br [[WORK:bb[0-9]+]]([[LIFETIME]] : $C) // CHECK: [[WORK]]([[LIFETIME_2:%[^,]+]] : @reborrow @guaranteed $C): +// CHECK: [[BF:%.*]] = borrowed [[LIFETIME_2]] // CHECK: cond_br undef, [[LEFT:bb[0-9]+]], [[RIGHT:bb[0-9]+]] // CHECK: [[LEFT]]: -// CHECK: end_borrow [[LIFETIME_2]] +// CHECK: end_borrow [[BF]] // CHECK: br [[EXIT:bb[0-9]+]] // CHECK: [[RIGHT]]: -// CHECK: end_borrow [[LIFETIME_2]] +// CHECK: end_borrow [[BF]] // CHECK: br [[EXIT]] // CHECK: [[EXIT]]: // CHECK: return [[INSTANCE]] diff --git a/test/SILOptimizer/sil_combine_ossa.sil b/test/SILOptimizer/sil_combine_ossa.sil index bf88a0d37c040..f6aba80dc3124 100644 --- a/test/SILOptimizer/sil_combine_ossa.sil +++ b/test/SILOptimizer/sil_combine_ossa.sil @@ -1,8 +1,8 @@ -// RUN: %target-sil-opt -enable-copy-propagation=requested-passes-only -enable-objc-interop -enforce-exclusivity=none -enable-sil-verify-all %s -sil-combine | %FileCheck %s -// RUN: %target-sil-opt -enable-copy-propagation=requested-passes-only -enable-objc-interop -enforce-exclusivity=none -enable-sil-verify-all %s -sil-combine -generic-specializer | %FileCheck %s --check-prefix=CHECK_FORWARDING_OWNERSHIP_KIND +// RUN: %target-sil-opt -enable-copy-propagation=requested-passes-only -enable-objc-interop -enforce-exclusivity=none -enable-sil-verify-all %s -update-borrowed-from -sil-combine | %FileCheck %s +// RUN: %target-sil-opt -enable-copy-propagation=requested-passes-only -enable-objc-interop -enforce-exclusivity=none -enable-sil-verify-all %s -update-borrowed-from -sil-combine -generic-specializer | %FileCheck %s --check-prefix=CHECK_FORWARDING_OWNERSHIP_KIND // // FIXME: check copy-propagation output instead once it is the default mode -// RUN: %target-sil-opt -enable-objc-interop -enforce-exclusivity=none -enable-sil-verify-all %s -sil-combine -enable-copy-propagation -enable-lexical-lifetimes=false | %FileCheck %s --check-prefix=CHECK_COPYPROP +// RUN: %target-sil-opt -enable-objc-interop -enforce-exclusivity=none -enable-sil-verify-all %s -update-borrowed-from -sil-combine -enable-copy-propagation -enable-lexical-lifetimes=false | %FileCheck %s --check-prefix=CHECK_COPYPROP // REQUIRES: swift_in_compiler diff --git a/test/SILOptimizer/simplify_cfg_checkcast.sil b/test/SILOptimizer/simplify_cfg_checkcast.sil index 2ef0599f73e2a..9ed123989f104 100644 --- a/test/SILOptimizer/simplify_cfg_checkcast.sil +++ b/test/SILOptimizer/simplify_cfg_checkcast.sil @@ -1,5 +1,5 @@ -// RUN: %target-sil-opt -enable-sil-verify-all %s -jumpthread-simplify-cfg -// RUN: %target-sil-opt -enable-sil-verify-all %s -jumpthread-simplify-cfg -debug-only=sil-simplify-cfg +// RUN: %target-sil-opt -enable-sil-verify-all %s -update-borrowed-from -jumpthread-simplify-cfg +// RUN: %target-sil-opt -enable-sil-verify-all %s -update-borrowed-from -jumpthread-simplify-cfg -debug-only=sil-simplify-cfg // // REQUIRES: asserts diff --git a/test/SILOptimizer/simplify_cfg_ossa_bbargs.sil b/test/SILOptimizer/simplify_cfg_ossa_bbargs.sil index 1cc63cd1e184f..5f3aa0d7109ce 100644 --- a/test/SILOptimizer/simplify_cfg_ossa_bbargs.sil +++ b/test/SILOptimizer/simplify_cfg_ossa_bbargs.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -test-runner %s | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -test-runner %s | %FileCheck %s import Builtin diff --git a/test/SILOptimizer/simplify_cfg_ossa_disabled.sil b/test/SILOptimizer/simplify_cfg_ossa_disabled.sil index eccacaff9d758..39d700154e259 100644 --- a/test/SILOptimizer/simplify_cfg_ossa_disabled.sil +++ b/test/SILOptimizer/simplify_cfg_ossa_disabled.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-objc-interop -enforce-exclusivity=none -enable-sil-verify-all %s -jumpthread-simplify-cfg | %FileCheck %s +// RUN: %target-sil-opt -enable-objc-interop -enforce-exclusivity=none -enable-sil-verify-all %s -update-borrowed-from -jumpthread-simplify-cfg | %FileCheck %s // // These tests case are converted to OSSA form, but aren't yet // optimized in OSSA mode. Move them to one of these files when diff --git a/test/SILOptimizer/simplify_cfg_ossa_simplify_branch.sil b/test/SILOptimizer/simplify_cfg_ossa_simplify_branch.sil index 1c420c8c8ef4c..057532ce2af40 100644 --- a/test/SILOptimizer/simplify_cfg_ossa_simplify_branch.sil +++ b/test/SILOptimizer/simplify_cfg_ossa_simplify_branch.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -test-runner %s 2>&1 | %FileCheck %s +// RUN: %target-sil-opt -update-borrowed-from -test-runner %s 2>&1 | %FileCheck %s enum FakeOptional { case some(T) diff --git a/test/SILOptimizer/simplify_cfg_tryapply.sil b/test/SILOptimizer/simplify_cfg_tryapply.sil index 90ccd27d180a7..cdf920555eabd 100644 --- a/test/SILOptimizer/simplify_cfg_tryapply.sil +++ b/test/SILOptimizer/simplify_cfg_tryapply.sil @@ -1,4 +1,6 @@ -// RUN: %target-sil-opt -enable-sil-verify-all %s -simplify-cfg | %FileCheck %s +// RUN: %target-sil-opt -enable-sil-verify-all %s -update-borrowed-from -simplify-cfg | %FileCheck %s + +// REQUIRES: swift_in_compiler // Declare this SIL to be canonical because some tests break raw SIL // conventions. e.g. address-type block args. -enforce-exclusivity=none is also @@ -355,8 +357,9 @@ bb3(%res : $Builtin.Int32): // CHECK: [[NONE:%.*]] = enum $Optional, #Optional.none!enumelt // CHECK: br bb3([[NONE]] : $Optional) // CHECK: bb3([[REWRAP:%.*]] : @guaranteed $Optional): +// CHECK: [[BF:%.*]] = borrowed [[REWRAP]] : $Optional from (%1 : $E, %0 : $Optional) // CHECK: [[ECAST:%.*]] = upcast [[EXTR1]] : $E to $B -// CHECK: [[OUTTUP:%.*]] = tuple ([[REWRAP]] : $Optional, [[ECAST]] : $B) +// CHECK: [[OUTTUP:%.*]] = tuple ([[BF]] : $Optional, [[ECAST]] : $B) // CHECK: apply %{{.*}}([[OUTTUP]]) : $@convention(thin) (@guaranteed (Optional, B)) -> Int // CHECK: return // CHECK-LABEL: } // end sil function 'replaceTryApplyGuaranteedTuple' diff --git a/test/SILOptimizer/simplify_unchecked_switch_enum.sil b/test/SILOptimizer/simplify_unchecked_switch_enum.sil index aab3aedc8a609..310ef60191507 100644 --- a/test/SILOptimizer/simplify_unchecked_switch_enum.sil +++ b/test/SILOptimizer/simplify_unchecked_switch_enum.sil @@ -75,8 +75,9 @@ bb2(%4 : @owned $E): // CHECK: bb0(%0 : @guaranteed $String): // CHECK-NEXT: br bb1(%0 : $String) // CHECK: bb1(%2 : @guaranteed $String): -// CHECK-NEXT: %3 = copy_value %2 -// CHECK-NEXT: return %3 : $String +// CHECK-NEXT: %3 = borrowed %2 : $String from () +// CHECK-NEXT: %4 = copy_value %3 +// CHECK-NEXT: return %4 : $String // CHECK-NOT: bb2 // CHECK: } // end sil function 'guaranteed_ossa' sil [ossa] @guaranteed_ossa : $@convention(thin) (@guaranteed String) -> @owned String { @@ -115,7 +116,8 @@ bb2(%7 : @owned $E): // CHECK-NEXT: end_borrow // CHECK-NEXT: br bb1(%0 : $String) // CHECK: bb1([[A:%.*]] : @guaranteed $String): -// CHECK-NEXT: [[C:%.*]] = copy_value [[A]] +// CHECK-NEXT: [[B:%.*]] = borrowed [[A]] : $String from () +// CHECK-NEXT: [[C:%.*]] = copy_value [[B]] // CHECK-NEXT: return [[C]] : $String // CHECK-NOT: bb2 // CHECK: } // end sil function 'guaranteed_with_additional_uses' diff --git a/test/SILOptimizer/temp_rvalue_opt_ossa.sil b/test/SILOptimizer/temp_rvalue_opt_ossa.sil index ca5052021972a..94f820a9555fe 100644 --- a/test/SILOptimizer/temp_rvalue_opt_ossa.sil +++ b/test/SILOptimizer/temp_rvalue_opt_ossa.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-sil-verify-all %s -temp-rvalue-opt | %FileCheck %s +// RUN: %target-sil-opt -enable-sil-verify-all %s -update-borrowed-from -temp-rvalue-opt | %FileCheck %s // REQUIRES: swift_in_compiler diff --git a/test/Serialization/ossa_sil.sil b/test/Serialization/ossa_sil.sil index d4604478eb823..343aa7eba81cb 100644 --- a/test/Serialization/ossa_sil.sil +++ b/test/Serialization/ossa_sil.sil @@ -19,3 +19,31 @@ bb0(%0 : @guaranteed $AnyObject): return %2 : $X } +// CHECK-LABEL: sil [serialized] [canonical] [ossa] @reborrowTest : +// CHECK: bb1: +// CHECK-NEXT: {{%.*}} = begin_borrow %0 : $X +// CHECK: bb2: +// CHECK-NEXT: {{%.*}} = begin_borrow %0 : $X +// CHECK: bb3([[ARG:%.*]] : @reborrow @guaranteed $X): +// CHECK-NEXT: {{%.*}} = borrowed [[ARG]] : $X from (%0 : $X) +// CHECK: } // end sil function 'reborrowTest' +sil [serialized] [ossa] @reborrowTest : $@convention(thin) (@owned X) -> () { +bb0(%0 : @owned $X): + cond_br undef, bb1, bb2 + +bb1: + %b1 = begin_borrow %0 : $X + br bb3(%b1 : $X) + +bb2: + %b2 = begin_borrow %0 : $X + br bb3(%b2 : $X) + +bb3(%r : @reborrow @guaranteed $X): + %f = borrowed %r : $X from (%0 : $X) + end_borrow %f : $X + destroy_value %0 : $X + %9999 = tuple() + return %9999 : $() +} +