tidying, more comments

Author: huntergr-arm

llvm/include/llvm/Analysis/IVDescriptors.h

@@ -178,19 +178,12 @@ class RecurrenceDescriptor {
   /// Returns a struct describing whether the instruction is either a
   ///   Select(ICmp(A, B), X, Y), or
   ///   Select(FCmp(A, B), X, Y)
-  /// where one of (X, Y) is an increasing (FindLast) or decreasing (FindFirst)
-  /// loop induction variable, and the other is a PHI value.
-  // TODO: Support non-monotonic variable. FindLast does not need be restricted
-  // to increasing loop induction variables.
-  LLVM_ABI static InstDesc isFindIVPattern(RecurKind Kind, Loop *TheLoop,
-                                           PHINode *OrigPhi, Instruction *I,
-                                           ScalarEvolution &SE);
-
-  /// Returns a struct describing whether the instruction is of the form
-  ///  Select(Cmp(A, B), X, Y)
-  /// where one of (X, Y) is the Phi value and the other is an arbitrary value.
-  LLVM_ABI static InstDesc isFindLastPattern(Instruction *I, PHINode *Phi,
-                                             Loop *TheLoop);
+  /// where one of (X, Y) is an increasing (FindLastIV) or decreasing
+  /// (FindFirstIV) loop induction variable or an (FindLast) arbitrary integer
+  /// value, and the other is a PHI value.
+  LLVM_ABI static InstDesc isFindPattern(RecurKind Kind, Loop *TheLoop,
+                                         PHINode *OrigPhi, Instruction *I,
+                                         ScalarEvolution &SE);
 
   /// Returns a struct describing if the instruction is a
   /// Select(FCmp(X, Y), (Z = X op PHINode), PHINode) instruction pattern.
@@ -315,7 +308,8 @@ class RecurrenceDescriptor {
   }
 
   /// Returns true if the recurrence kind is of the form
-  ///   select(cmp(),x,y) where one of (x,y) is an arbitrary value.
+  ///   select(cmp(),x,y) where one of (x,y) is an arbitrary value and the
+  ///   other is a recurrence.
   static bool isFindLastRecurrenceKind(RecurKind Kind) {
     return Kind == RecurKind::FindLast;
   }

llvm/lib/Analysis/IVDescriptors.cpp

@@ -17,9 +17,11 @@
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Dominators.h"
+#include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/KnownBits.h"
 
@@ -693,9 +695,9 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
 // value of the data type or a non-constant value by using mask and multiple
 // reduction operations.
 RecurrenceDescriptor::InstDesc
-RecurrenceDescriptor::isFindIVPattern(RecurKind Kind, Loop *TheLoop,
-                                      PHINode *OrigPhi, Instruction *I,
-                                      ScalarEvolution &SE) {
+RecurrenceDescriptor::isFindPattern(RecurKind Kind, Loop *TheLoop,
+                                    PHINode *OrigPhi, Instruction *I,
+                                    ScalarEvolution &SE) {
   // TODO: Support the vectorization of FindLastIV when the reduction phi is
   // used by more than one select instruction. This vectorization is only
   // performed when the SCEV of each increasing induction variable used by the
@@ -721,16 +723,14 @@ RecurrenceDescriptor::isFindIVPattern(RecurKind Kind, Loop *TheLoop,
       return InstDesc(false, I);
 
     // FIXME: Support more complex patterns, including multiple selects.
-    // Phi or Select must be used only outside the loop,
-    // except for each other.
-    if (!all_of(I->users(), [OrigPhi, TheLoop](User *U) {
-          if (U == OrigPhi)
-            return true;
-          if (auto *UI = dyn_cast<Instruction>(U))
-            return !TheLoop->contains(UI);
-          return false;
-        }))
+    // The Select must be used only outside the loop and by the PHI.
+    for (User *U : I->users()) {
+      if (U == OrigPhi)
+        continue;
+      if (auto *UI = dyn_cast<Instruction>(U); UI && !TheLoop->contains(UI))
+        continue;
       return InstDesc(false, I);
+    }
 
     return InstDesc(I, RecurKind::FindLast);
   }
@@ -944,7 +944,7 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
         Kind == RecurKind::Sub || Kind == RecurKind::AddChainWithSubs)
       return isConditionalRdxPattern(I);
     if ((isFindIVRecurrenceKind(Kind) || isFindLastRecurrenceKind(Kind)) && SE)
-      return isFindIVPattern(Kind, L, OrigPhi, I, *SE);
+      return isFindPattern(Kind, L, OrigPhi, I, *SE);
     [[fallthrough]];
   case Instruction::FCmp:
   case Instruction::ICmp:

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

@@ -1033,7 +1033,7 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
     break;
   }
   case Intrinsic::experimental_vector_extract_last_active:
-    if (ST->isSVEAvailable()) {
+    if (ST->isSVEorStreamingSVEAvailable()) {
       auto [LegalCost, _] = getTypeLegalizationCost(ICA.getArgTypes()[0]);
       // This should turn into chained clastb instructions.
       return LegalCost;

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -10011,8 +10011,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   IC = UserIC > 0 ? UserIC : IC;
 
   // FIXME: Enable interleaving for last_active reductions.
-  if (any_of(LVL.getReductionVars(), [&](auto &Reduction) -> bool {
-        const RecurrenceDescriptor &RdxDesc = Reduction.second;
+  if (any_of(make_second_range(LVL.getReductionVars()), [&](auto &RdxDesc) {
         return RecurrenceDescriptor::isFindLastRecurrenceKind(
             RdxDesc.getRecurrenceKind());
       })) {

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -1772,7 +1772,7 @@ struct LLVM_ABI_FOR_TEST VPWidenSelectRecipe : public VPRecipeWithIRFlags,
   }
 };
 
-/// A recipe for selecting whole vector values.
+/// A recipe for selecting whole vector values based on a scalar condition.
 struct VPWidenSelectVectorRecipe : public VPRecipeWithIRFlags {
   VPWidenSelectVectorRecipe(ArrayRef<VPValue *> Operands)
       : VPRecipeWithIRFlags(VPDef::VPWidenSelectVectorSC, Operands) {}

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -4532,7 +4532,6 @@ void VPlanTransforms::addExitUsersForFirstOrderRecurrences(VPlan &Plan,
   }
 }
 
-/// Change CSA reductions to save the appropriate state.
 void VPlanTransforms::convertFindLastRecurrences(
     VPlan &Plan, VPRecipeBuilder &RecipeBuilder,
     LoopVectorizationLegality *Legal) {
@@ -4542,6 +4541,22 @@ void VPlanTransforms::convertFindLastRecurrences(
   if (Plan.hasScalarVFOnly())
     return;
 
+  // We want to create the following nodes:
+  // vec.body:
+  //   mask.phi = phi <VF x i1> [ all.false, vec.ph ], [ new.mask, vec.body ]
+  //   ...data.phi already exists, but needs updating...
+  //   data.phi = phi <VF x Ty> [ default.val, vec.ph ], [ new.data, vec.body ]
+  //
+  //   ...'data' and 'compare' created by existing nodes...
+  //
+  //   any_active = i1 any_of_reduction(compare)
+  //   new.mask = select any_active, compare, mask.phi
+  //   new.data = select any_active, data, data.phi
+  //
+  // middle.block:
+  //   ...the extract already exists, but needs updating...
+  //   result = extract-last-active new.data, new.mask, default.val
+
   for (const auto &[Phi, RdxDesc] : Legal->getReductionVars()) {
     if (RecurrenceDescriptor::isFindLastRecurrenceKind(
             RdxDesc.getRecurrenceKind())) {

llvm/lib/Transforms/Vectorize/VPlanTransforms.h

@@ -380,7 +380,8 @@ struct VPlanTransforms {
   static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range);
 
   /// Change FindLast reductions to save the appropriate state using selects
-  /// for entire vectors for both the data and the mask..
+  /// for entire vectors for both the latest mask containing at least one active
+  /// element and the corresponding data vector.
   static void convertFindLastRecurrences(VPlan &Plan,
                                          VPRecipeBuilder &RecipeBuilder,
                                          LoopVectorizationLegality *Legal);