@@ -9015,6 +9015,9 @@ void BoUpSLP::reorderInputsAccordingToOpcode(
90159015}
90169016
90179017Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
9018+ auto &Res = EntryToLastInstruction.FindAndConstruct(E);
9019+ if (Res.second)
9020+ return *Res.second;
90189021 // Get the basic block this bundle is in. All instructions in the bundle
90199022 // should be in this block (except for extractelement-like instructions with
90209023 // constant indeces).
@@ -9029,7 +9032,7 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
90299032 isVectorLikeInstWithConstOps(I);
90309033 }));
90319034
9032- auto && FindLastInst = [E, Front, this, &BB ]() {
9035+ auto FindLastInst = [& ]() {
90339036 Instruction *LastInst = Front;
90349037 for (Value *V : E->Scalars) {
90359038 auto *I = dyn_cast<Instruction>(V);
@@ -9065,7 +9068,7 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
90659068 return LastInst;
90669069 };
90679070
9068- auto && FindFirstInst = [E, Front, this ]() {
9071+ auto FindFirstInst = [& ]() {
90699072 Instruction *FirstInst = Front;
90709073 for (Value *V : E->Scalars) {
90719074 auto *I = dyn_cast<Instruction>(V);
@@ -9105,7 +9108,6 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
91059108 if (doesNotNeedToSchedule(E->Scalars) ||
91069109 (E->State != TreeEntry::NeedToGather &&
91079110 all_of(E->Scalars, isVectorLikeInstWithConstOps))) {
9108- Instruction *InsertInst;
91099111 if ((E->getOpcode() == Instruction::GetElementPtr &&
91109112 any_of(E->Scalars,
91119113 [](Value *V) {
@@ -9114,15 +9116,12 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
91149116 all_of(E->Scalars, [](Value *V) {
91159117 return !isVectorLikeInstWithConstOps(V) && isUsedOutsideBlock(V);
91169118 }))
9117- InsertInst = FindLastInst();
9119+ Res.second = FindLastInst();
91189120 else
9119- InsertInst = FindFirstInst();
9120- return *InsertInst ;
9121+ Res.second = FindFirstInst();
9122+ return *Res.second ;
91219123 }
91229124
9123- // The last instruction in the bundle in program order.
9124- Instruction *LastInst = nullptr;
9125-
91269125 // Find the last instruction. The common case should be that BB has been
91279126 // scheduled, and the last instruction is VL.back(). So we start with
91289127 // VL.back() and iterate over schedule data until we reach the end of the
@@ -9135,7 +9134,7 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
91359134 if (Bundle && Bundle->isPartOfBundle())
91369135 for (; Bundle; Bundle = Bundle->NextInBundle)
91379136 if (Bundle->OpValue == Bundle->Inst)
9138- LastInst = Bundle->Inst;
9137+ Res.second = Bundle->Inst;
91399138 }
91409139
91419140 // LastInst can still be null at this point if there's either not an entry
@@ -9156,15 +9155,15 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
91569155 // not ideal. However, this should be exceedingly rare since it requires that
91579156 // we both exit early from buildTree_rec and that the bundle be out-of-order
91589157 // (causing us to iterate all the way to the end of the block).
9159- if (!LastInst )
9160- LastInst = FindLastInst();
9161- assert(LastInst && "Failed to find last instruction in bundle");
9162- return *LastInst ;
9158+ if (!Res.second )
9159+ Res.second = FindLastInst();
9160+ assert(Res.second && "Failed to find last instruction in bundle");
9161+ return *Res.second ;
91639162}
91649163
91659164void BoUpSLP::setInsertPointAfterBundle(const TreeEntry *E) {
91669165 auto *Front = E->getMainOp();
9167- Instruction *LastInst = EntryToLastInstruction.lookup (E);
9166+ Instruction *LastInst = &getLastInstructionInBundle (E);
91689167 assert(LastInst && "Failed to find last instruction in bundle");
91699168 // If the instruction is PHI, set the insert point after all the PHIs.
91709169 bool IsPHI = isa<PHINode>(LastInst);
@@ -9685,7 +9684,7 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx) {
96859684 IRBuilder<>::InsertPointGuard Guard(Builder);
96869685 if (E->getOpcode() != Instruction::InsertElement &&
96879686 E->getOpcode() != Instruction::PHI) {
9688- Instruction *LastInst = EntryToLastInstruction.lookup (E);
9687+ Instruction *LastInst = &getLastInstructionInBundle (E);
96899688 assert(LastInst && "Failed to find last instruction in bundle");
96909689 Builder.SetInsertPoint(LastInst);
96919690 }
@@ -10704,18 +10703,9 @@ Value *BoUpSLP::vectorizeTree(
1070410703 for (auto &BSIter : BlocksSchedules) {
1070510704 scheduleBlock(BSIter.second.get());
1070610705 }
10707-
10708- // Pre-gather last instructions.
10709- for (const std::unique_ptr<TreeEntry> &E : VectorizableTree) {
10710- if ((E->State == TreeEntry::NeedToGather &&
10711- (!E->getMainOp() || E->Idx > 0)) ||
10712- (E->State != TreeEntry::NeedToGather &&
10713- E->getOpcode() == Instruction::ExtractValue) ||
10714- E->getOpcode() == Instruction::InsertElement)
10715- continue;
10716- Instruction *LastInst = &getLastInstructionInBundle(E.get());
10717- EntryToLastInstruction.try_emplace(E.get(), LastInst);
10718- }
10706+ // Clean Entry-to-LastInstruction table. It can be affected after scheduling,
10707+ // need to rebuild it.
10708+ EntryToLastInstruction.clear();
1071910709
1072010710 Builder.SetInsertPoint(ReductionRoot ? ReductionRoot
1072110711 : &F->getEntryBlock().front());
0 commit comments