[InstCombine] Support well-defined recurrences in isGuaranteedNotToBeUndefOrPoison

This function currently doesn't detect well-defined recurrences that
can't produce undef or poison. This prevents instcombine from pushing
freezes through GEPs to the ptr which may be undef or poison, when the
offset is a well-defined recurrence, as the tests added in #145541
demonstrate.

This patch fixes this by pulling existing code from
foldFreezeIntoRecurrence in instcombine, which can detect whether a PHI
produces undef/poison, so it can be reused by
isGuaranteedNotToBeUndefOrPoison in ValueTracking.

Author: c-rhodes

llvm/include/llvm/Analysis/ValueTracking.h

@@ -773,6 +773,14 @@ LLVM_ABI bool canCreatePoison(const Operator *Op,
 /// impliesPoison returns true.
 LLVM_ABI bool impliesPoison(const Value *ValAssumedPoison, const Value *V);
 
+/// Detect if PN is a recurrence with a start value and some number of backedge
+/// values. We'll check whether we can push the freeze through the backedge
+/// values (possibly dropping poison flags along the way) until we reach the
+/// phi again. In that case, we can move the freeze to the start value.
+LLVM_ABI Use *canFoldFreezeIntoRecurrence(
+    PHINode *PN, DominatorTree *DT, bool &StartNeedsFreeze,
+    SmallVectorImpl<Instruction *> *DropFlags = nullptr);
+
 /// Return true if this function can prove that V does not have undef bits
 /// and is never poison. If V is an aggregate value or vector, check whether
 /// all elements (except padding) are not undef or poison.

llvm/lib/Analysis/ValueTracking.cpp

@@ -7569,6 +7569,66 @@ bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
 
 static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
 
+Use *llvm::canFoldFreezeIntoRecurrence(
+    PHINode *PN, DominatorTree *DT, bool &StartNeedsFreeze,
+    SmallVectorImpl<Instruction *> *DropFlags) {
+  // Detect whether this is a recurrence with a start value and some number of
+  // backedge values. We'll check whether we can push the freeze through the
+  // backedge values (possibly dropping poison flags along the way) until we
+  // reach the phi again. In that case, we can move the freeze to the start
+  // value.
+  Use *StartU = nullptr;
+  SmallVector<Value *> Worklist;
+  for (Use &U : PN->incoming_values()) {
+    if (DT && DT->dominates(PN->getParent(), PN->getIncomingBlock(U))) {
+      // Add backedge value to worklist.
+      Worklist.push_back(U.get());
+      continue;
+    }
+
+    // Don't bother handling multiple start values.
+    if (StartU)
+      return nullptr;
+    StartU = &U;
+  }
+
+  if (!StartU || Worklist.empty())
+    return nullptr; // Not a recurrence.
+
+  Value *StartV = StartU->get();
+  BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
+  StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(StartV);
+  // We can't insert freeze if the start value is the result of the
+  // terminator (e.g. an invoke).
+  if (StartNeedsFreeze && StartBB->getTerminator() == StartV)
+    return nullptr;
+
+  SmallPtrSet<Value *, 32> Visited;
+  while (!Worklist.empty()) {
+    Value *V = Worklist.pop_back_val();
+    if (!Visited.insert(V).second)
+      continue;
+
+    if (Visited.size() > 32)
+      return nullptr; // Limit the total number of values we inspect.
+
+    // Assume that PN is non-poison, because it will be after the transform.
+    if (V == PN || isGuaranteedNotToBeUndefOrPoison(V))
+      continue;
+
+    Instruction *I = dyn_cast<Instruction>(V);
+    if (!I || canCreateUndefOrPoison(cast<Operator>(I),
+                                     /*ConsiderFlagsAndMetadata*/ false))
+      return nullptr;
+
+    if (DropFlags)
+      DropFlags->push_back(I);
+    append_range(Worklist, I->operands());
+  }
+
+  return StartU;
+}
+
 static bool isGuaranteedNotToBeUndefOrPoison(
     const Value *V, AssumptionCache *AC, const Instruction *CtxI,
     const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
@@ -7657,6 +7717,13 @@ static bool isGuaranteedNotToBeUndefOrPoison(
       }
       if (IsWellDefined)
         return true;
+
+      bool StartNeedsFreeze;
+      if (canFoldFreezeIntoRecurrence(const_cast<PHINode *>(PN),
+                                      const_cast<DominatorTree *>(DT),
+                                      StartNeedsFreeze) &&
+          !StartNeedsFreeze)
+        return true;
     } else if (!::canCreateUndefOrPoison(Opr, Kind,
                                          /*ConsiderFlagsAndMetadata*/ true) &&
                all_of(Opr->operands(), OpCheck))

llvm/lib/Transforms/InstCombine/InstructionCombining.cpp

@@ -4933,7 +4933,8 @@ InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating(FreezeInst &OrigFI) {
   // poison.
   Value *MaybePoisonOperand = nullptr;
   for (Value *V : OrigOpInst->operands()) {
-    if (isa<MetadataAsValue>(V) || isGuaranteedNotToBeUndefOrPoison(V) ||
+    if (isa<MetadataAsValue>(V) ||
+        isGuaranteedNotToBeUndefOrPoison(V, &AC, &OrigFI, &DT) ||
         // Treat identical operands as a single operand.
         (MaybePoisonOperand && MaybePoisonOperand == V))
       continue;
@@ -4959,64 +4960,19 @@ InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating(FreezeInst &OrigFI) {
 
 Instruction *InstCombinerImpl::foldFreezeIntoRecurrence(FreezeInst &FI,
                                                         PHINode *PN) {
-  // Detect whether this is a recurrence with a start value and some number of
-  // backedge values. We'll check whether we can push the freeze through the
-  // backedge values (possibly dropping poison flags along the way) until we
-  // reach the phi again. In that case, we can move the freeze to the start
-  // value.
-  Use *StartU = nullptr;
-  SmallVector<Value *> Worklist;
-  for (Use &U : PN->incoming_values()) {
-    if (DT.dominates(PN->getParent(), PN->getIncomingBlock(U))) {
-      // Add backedge value to worklist.
-      Worklist.push_back(U.get());
-      continue;
-    }
-
-    // Don't bother handling multiple start values.
-    if (StartU)
-      return nullptr;
-    StartU = &U;
-  }
-
-  if (!StartU || Worklist.empty())
-    return nullptr; // Not a recurrence.
-
-  Value *StartV = StartU->get();
-  BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
-  bool StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(StartV);
-  // We can't insert freeze if the start value is the result of the
-  // terminator (e.g. an invoke).
-  if (StartNeedsFreeze && StartBB->getTerminator() == StartV)
-    return nullptr;
-
-  SmallPtrSet<Value *, 32> Visited;
+  bool StartNeedsFreeze;
   SmallVector<Instruction *> DropFlags;
-  while (!Worklist.empty()) {
-    Value *V = Worklist.pop_back_val();
-    if (!Visited.insert(V).second)
-      continue;
-
-    if (Visited.size() > 32)
-      return nullptr; // Limit the total number of values we inspect.
-
-    // Assume that PN is non-poison, because it will be after the transform.
-    if (V == PN || isGuaranteedNotToBeUndefOrPoison(V))
-      continue;
-
-    Instruction *I = dyn_cast<Instruction>(V);
-    if (!I || canCreateUndefOrPoison(cast<Operator>(I),
-                                     /*ConsiderFlagsAndMetadata*/ false))
-      return nullptr;
-
-    DropFlags.push_back(I);
-    append_range(Worklist, I->operands());
-  }
+  Use *StartU =
+      canFoldFreezeIntoRecurrence(PN, &DT, StartNeedsFreeze, &DropFlags);
+  if (!StartU)
+    return nullptr;
 
   for (Instruction *I : DropFlags)
     I->dropPoisonGeneratingAnnotations();
 
   if (StartNeedsFreeze) {
+    Value *StartV = StartU->get();
+    BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
     Builder.SetInsertPoint(StartBB->getTerminator());
     Value *FrozenStartV = Builder.CreateFreeze(StartV,
                                                StartV->getName() + ".fr");

llvm/test/Transforms/Attributor/dereferenceable-1.ll

@@ -95,7 +95,7 @@ define void @deref_phi_growing(ptr dereferenceable(4000) %a) {
 ; CHECK:       for.cond:
 ; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    [[A_ADDR_0:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[INCDEC_PTR:%.*]], [[FOR_INC]] ]
-; CHECK-NEXT:    call void @deref_phi_user(ptr nonnull [[A_ADDR_0]])
+; CHECK-NEXT:    call void @deref_phi_user(ptr noundef nonnull [[A_ADDR_0]])
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[A_ADDR_0]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[VAL]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -146,7 +146,7 @@ define void @deref_phi_shrinking(ptr dereferenceable(4000) %a) {
 ; CHECK:       for.cond:
 ; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    [[A_ADDR_0:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[INCDEC_PTR:%.*]], [[FOR_INC]] ]
-; CHECK-NEXT:    call void @deref_phi_user(ptr nonnull [[A_ADDR_0]])
+; CHECK-NEXT:    call void @deref_phi_user(ptr noundef nonnull [[A_ADDR_0]])
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[A_ADDR_0]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[VAL]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]

llvm/test/Transforms/Attributor/value-simplify.ll

@@ -626,7 +626,7 @@ define internal ptr @test_byval2(ptr byval(%struct.X) %a) {
 ; CHECK-NEXT:    [[A_PRIV:%.*]] = alloca [[STRUCT_X:%.*]], align 8
 ; CHECK-NEXT:    store ptr [[TMP0]], ptr [[A_PRIV]], align 8
 ; CHECK-NEXT:    call void @sync()
-; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[A_PRIV]], align 8
+; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[A_PRIV]], align 8, !invariant.load [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret ptr [[L]]
 ;
   call void @sync()
@@ -1359,7 +1359,7 @@ define internal i32 @ret_speculatable_expr(ptr %mem, i32 %a2) {
 ; CGSCC-SAME: (i32 [[TMP0:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:    [[MEM_PRIV:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 [[TMP0]], ptr [[MEM_PRIV]], align 4
-; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr [[MEM_PRIV]], align 4
+; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr [[MEM_PRIV]], align 4, !invariant.load [[META0]]
 ; CGSCC-NEXT:    [[MUL:%.*]] = mul i32 [[L]], 13
 ; CGSCC-NEXT:    [[ADD:%.*]] = add i32 [[MUL]], 7
 ; CGSCC-NEXT:    ret i32 [[ADD]]
@@ -1709,3 +1709,7 @@ define i32 @readExtInitZeroInit() {
 ; CGSCC: attributes #[[ATTR17]] = { nosync }
 ; CGSCC: attributes #[[ATTR18]] = { nounwind }
 ;.
+; TUNIT: [[META0]] = !{}
+;.
+; CGSCC: [[META0]] = !{}
+;.

llvm/test/Transforms/InstCombine/freeze-landingpad.ll

@@ -9,19 +9,18 @@ define i32 @propagate_freeze_in_landingpad() personality ptr null {
 ; CHECK:       invoke.bb1:
 ; CHECK-NEXT:    [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[NORMAL_RETURN:%.*]] ]
 ; CHECK-NEXT:    [[RES0:%.*]] = invoke i32 @foo()
-; CHECK-NEXT:    to label [[INVOKE_BB2:%.*]] unwind label [[EXCEPTIONAL_RETURN:%.*]]
+; CHECK-NEXT:            to label [[INVOKE_BB2:%.*]] unwind label [[EXCEPTIONAL_RETURN:%.*]]
 ; CHECK:       invoke.bb2:
 ; CHECK-NEXT:    [[RES1:%.*]] = invoke i32 @foo()
-; CHECK-NEXT:    to label [[NORMAL_RETURN]] unwind label [[EXCEPTIONAL_RETURN]]
+; CHECK-NEXT:            to label [[NORMAL_RETURN]] unwind label [[EXCEPTIONAL_RETURN]]
 ; CHECK:       normal_return:
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[X]], 1
 ; CHECK-NEXT:    br label [[INVOKE_BB1]]
 ; CHECK:       exceptional_return:
 ; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X]], [[INVOKE_BB1]] ], [ 0, [[INVOKE_BB2]] ]
 ; CHECK-NEXT:    [[LANDING_PAD:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT:    cleanup
-; CHECK-NEXT:    [[FR:%.*]] = freeze i32 [[PHI]]
-; CHECK-NEXT:    [[RES:%.*]] = shl i32 [[FR]], 1
+; CHECK-NEXT:            cleanup
+; CHECK-NEXT:    [[RES:%.*]] = shl nuw i32 [[PHI]], 1
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
 entry:

llvm/test/Transforms/InstCombine/freeze.ll

@@ -889,17 +889,17 @@ exit:                                             ; preds = %loop
 }
 
 ; The recurrence for the GEP offset can't produce poison so the freeze should
-; be pushed through to the ptr, but this is not currently supported.
+; be pushed through to the ptr.
 define void @fold_phi_gep_phi_offset(ptr %init, ptr %end, i64 noundef %n) {
 ; CHECK-LABEL: @fold_phi_gep_phi_offset(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INIT:%.*]] = freeze ptr [[INIT1:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[INIT]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFF:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFF_NEXT]] = shl i64 [[OFF]], 3
-; CHECK-NEXT:    [[I_NEXT:%.*]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
-; CHECK-NEXT:    [[I_NEXT_FR]] = freeze ptr [[I_NEXT]]
+; CHECK-NEXT:    [[I_NEXT_FR]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
@@ -921,18 +921,18 @@ exit:                                             ; preds = %loop
   ret void
 }
 
-; Offset is still guaranteed not to be poison, so the freeze could be moved
-; here if we strip inbounds from the GEP, but this is not currently supported.
+; Offset is still guaranteed not to be poison, so the freeze can be moved
+; here if we strip inbounds from the GEP.
 define void @fold_phi_gep_inbounds_phi_offset(ptr %init, ptr %end, i64 noundef %n) {
 ; CHECK-LABEL: @fold_phi_gep_inbounds_phi_offset(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INIT:%.*]] = freeze ptr [[INIT1:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[INIT]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFF:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFF_NEXT]] = shl i64 [[OFF]], 3
-; CHECK-NEXT:    [[I_NEXT:%.*]] = getelementptr inbounds i8, ptr [[I]], i64 [[OFF_NEXT]]
-; CHECK-NEXT:    [[I_NEXT_FR]] = freeze ptr [[I_NEXT]]
+; CHECK-NEXT:    [[I_NEXT_FR]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
@@ -994,7 +994,7 @@ define void @fold_phi_multiple_insts(i32 %init, i32 %n) {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT_FR]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[I_SQ:%.*]] = mul i32 [[I]], [[I]]
-; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I_SQ]], 1
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I_SQ]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
@@ -1127,7 +1127,7 @@ define void @fold_phi_invoke_noundef_start_value(i32 %n) personality ptr undef {
 ; CHECK-NEXT:            to label [[LOOP:%.*]] unwind label [[UNWIND:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       unwind:

llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll

@@ -16,13 +16,11 @@ define void @pr56282() {
 ; CHECK:       outer.header:
 ; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_IV_NEXT:%.*]], [[INNER_2:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OUTER_IV]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], -1
-; CHECK-NEXT:    [[XTRAITER:%.*]] = and i64 [[TMP1]], 7
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP2]], 7
+; CHECK-NEXT:    [[XTRAITER:%.*]] = and i64 [[TMP0]], 7
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[OUTER_IV]], 7
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[OUTER_MIDDLE_UNR_LCSSA:%.*]], label [[OUTER_HEADER_NEW:%.*]]
 ; CHECK:       outer.header.new:
-; CHECK-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[TMP1]], [[XTRAITER]]
+; CHECK-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[TMP0]], [[XTRAITER]]
 ; CHECK-NEXT:    br label [[INNER_1_HEADER:%.*]]
 ; CHECK:       inner.1.header:
 ; CHECK-NEXT:    [[INNER_1_IV:%.*]] = phi i64 [ 0, [[OUTER_HEADER_NEW]] ], [ [[INNER_1_IV_NEXT_7:%.*]], [[INNER_1_LATCH_7:%.*]] ]

llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll

@@ -662,8 +662,7 @@ define dso_local void @select_invariant_outer_loop(i1 noundef zeroext %cond, i32
 ; CHECK-NEXT:    [[I_021_US:%.*]] = phi i32 [ [[INC9_US:%.*]], [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US:%.*]] ], [ 0, [[FOR_COND1_PREHEADER_US_PREHEADER]] ]
 ; CHECK-NEXT:    [[REM_US:%.*]] = and i32 [[I_021_US]], 1
 ; CHECK-NEXT:    [[CMP5_US:%.*]] = icmp eq i32 [[REM_US]], 0
-; CHECK-NEXT:    [[CMP5_US_FR:%.*]] = freeze i1 [[CMP5_US]]
-; CHECK-NEXT:    br i1 [[CMP5_US_FR]], label [[FOR_COND1_PREHEADER_US_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_US_SPLIT:%.*]]
+; CHECK-NEXT:    br i1 [[CMP5_US]], label [[FOR_COND1_PREHEADER_US_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_US_SPLIT:%.*]]
 ; CHECK:       for.cond1.preheader.us.split.us:
 ; CHECK-NEXT:    br label [[FOR_BODY4_US_US:%.*]]
 ; CHECK:       for.body4.us.us: