diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceInsertion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceInsertion.swift
index 25549b521765e..11016e479ade6 100644
--- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceInsertion.swift
+++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceInsertion.swift
@@ -424,6 +424,10 @@ struct VariableIntroducerUseDefWalker : LifetimeDependenceUseDefValueWalker, Lif
     visitedValues.insert(value)
   }
 
+  mutating func needWalk(for address: Value) -> Bool {
+    visitedValues.insert(address)
+  }
+
   mutating func walkUp(newLifetime: Value) -> WalkResult {
     if newLifetime.type.isAddress {
       return walkUp(address: newLifetime)
diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift
index 41bec71987fce..355c1d33f0c53 100644
--- a/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift
+++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift
@@ -817,7 +817,10 @@ extension LifetimeDependenceDefUseWalker {
   }
 
   private mutating func walkDownAddressUses(of address: Value) -> WalkResult {
-    address.uses.ignoreTypeDependence.walk {
+    if !needWalk(for: address) {
+      return .continueWalk
+    }
+    return address.uses.ignoreTypeDependence.walk {
       return classifyAddress(operand: $0)
     }
   }
@@ -1199,6 +1202,8 @@ protocol LifetimeDependenceUseDefAddressWalker {
   // ignored.
   var isTrivialScope: Bool { get }
 
+  mutating func needWalk(for address: Value) -> Bool
+
   mutating func addressIntroducer(_ address: Value, access: AccessBaseAndScopes) -> WalkResult
 
   // The 'newLifetime' value is not forwarded to its uses. It may be a non-address or an address.
@@ -1221,6 +1226,9 @@ extension LifetimeDependenceUseDefAddressWalker {
   }
 
   mutating func walkUpDefault(address: Value, access: AccessBaseAndScopes) -> WalkResult {
+    if !needWalk(for: address) {
+      return .continueWalk
+    }
     if let beginAccess = access.innermostAccess {
       // Skip the access scope for unsafe[Mutable]Address. Treat it like a projection of 'self' rather than a separate
       // variable access.
@@ -1333,6 +1341,10 @@ struct LifetimeDependenceRootWalker : LifetimeDependenceUseDefValueWalker, Lifet
     visitedValues.insert(value)
   }
 
+  mutating func needWalk(for address: Value) -> Bool {
+    visitedValues.insert(address)
+  }
+
   mutating func walkUp(newLifetime: Value) -> WalkResult {
     if newLifetime.type.isAddress {
       return walkUp(address: newLifetime)