diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 3c367664a0988..86f5bc6c33d34 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2398,39 +2398,7 @@ void VPVectorPointerRecipe::print(raw_ostream &O, const Twine &Indent, #endif void VPBlendRecipe::execute(VPTransformState &State) { - assert(isNormalized() && "Expected blend to be normalized!"); - // We know that all PHIs in non-header blocks are converted into - // selects, so we don't have to worry about the insertion order and we - // can just use the builder. - // At this point we generate the predication tree. There may be - // duplications since this is a simple recursive scan, but future - // optimizations will clean it up. - - unsigned NumIncoming = getNumIncomingValues(); - - // Generate a sequence of selects of the form: - // SELECT(Mask3, In3, - // SELECT(Mask2, In2, - // SELECT(Mask1, In1, - // In0))) - // Note that Mask0 is never used: lanes for which no path reaches this phi and - // are essentially undef are taken from In0. - bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this); - Value *Result = nullptr; - for (unsigned In = 0; In < NumIncoming; ++In) { - // We might have single edge PHIs (blocks) - use an identity - // 'select' for the first PHI operand. - Value *In0 = State.get(getIncomingValue(In), OnlyFirstLaneUsed); - if (In == 0) - Result = In0; // Initialize with the first incoming value. - else { - // Select between the current value and the previous incoming edge - // based on the incoming mask. - Value *Cond = State.get(getMask(In), OnlyFirstLaneUsed); - Result = State.Builder.CreateSelect(Cond, In0, Result, "predphi"); - } - } - State.set(this, Result, OnlyFirstLaneUsed); + llvm_unreachable("VPBlendRecipe should be expanded by simplifyBlends"); } InstructionCost VPBlendRecipe::computeCost(ElementCount VF, diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 6a3b3e6e41955..bf5a6750f0fd6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2711,6 +2711,20 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan, continue; } + // Expand VPBlendRecipe into VPInstruction::Select + VPBuilder Builder(&R); + if (auto *Blend = dyn_cast(&R)) { + VPValue *Select = Blend->getIncomingValue(0); + for (unsigned I = 1; I != Blend->getNumIncomingValues(); ++I) { + Select = Builder.createSelect(Blend->getMask(I), + Blend->getIncomingValue(I), Select, + R.getDebugLoc(), "predphi"); + Select->setUnderlyingValue(Blend->getUnderlyingValue()); + } + Blend->replaceAllUsesWith(Select); + ToRemove.push_back(Blend); + } + if (auto *Expr = dyn_cast(&R)) { Expr->decompose(); ToRemove.push_back(Expr); @@ -2724,7 +2738,6 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan, // Expand WideIVStep. auto *VPI = cast(&R); - VPBuilder Builder(VPI); Type *IVTy = TypeInfo.inferScalarType(VPI); if (TypeInfo.inferScalarType(VectorStep) != IVTy) { Instruction::CastOps CastOp = IVTy->isFloatingPointTy()