[Flang] Loop Interchange

flang/include/flang/Lower/AbstractConverter.h

@@ -384,6 +384,10 @@ class AbstractConverter {
 
   virtual mlir::StateStack &getStateStack() = 0;
 
+  virtual void
+  genPermutatedLoops(llvm::ArrayRef<Fortran::lower::pft::Evaluation *> doStmts,
+                     Fortran::lower::pft::Evaluation *innermostDo) = 0;
+
 private:
   /// Options controlling lowering behavior.
   const Fortran::lower::LoweringOptions &loweringOptions;

flang/lib/Lower/Bridge.cpp

@@ -2197,6 +2197,124 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     // so no clean-up needs to be generated for these entities.
   }
 
+  void
+  genPermutatedLoops(llvm::ArrayRef<Fortran::lower::pft::Evaluation *> doStmts,
+                     Fortran::lower::pft::Evaluation *innermostDo) override {
+    // Fortran::lower::pft::Evaluation &eval = getEval();
+    // bool unstructuredContext = eval.lowerAsUnstructured();
+
+    llvm::SmallVector<mlir::Block *> headerBlocks;
+    llvm::SmallVector<IncrementLoopNestInfo, 1> loopInfos;
+
+    auto enterLoop = [&](Fortran::lower::pft::Evaluation &eval) {
+      bool unstructuredContext = eval.lowerAsUnstructured();
+
+      // Collect loop nest information.
+      // Generate begin loop code directly for infinite and while loops.
+      Fortran::lower::pft::Evaluation &doStmtEval =
+          eval.getFirstNestedEvaluation();
+      auto *doStmt = doStmtEval.getIf<Fortran::parser::NonLabelDoStmt>();
+      const auto &loopControl =
+          std::get<std::optional<Fortran::parser::LoopControl>>(doStmt->t);
+      mlir::Block *preheaderBlock = doStmtEval.block;
+      mlir::Block *beginBlock =
+          preheaderBlock ? preheaderBlock : builder->getBlock();
+      auto createNextBeginBlock = [&]() {
+        // Step beginBlock through unstructured preheader, header, and mask
+        // blocks, created in outermost to innermost order.
+        return beginBlock = beginBlock->splitBlock(beginBlock->end());
+      };
+      mlir::Block *headerBlock =
+          unstructuredContext ? createNextBeginBlock() : nullptr;
+      headerBlocks.push_back(headerBlock);
+      mlir::Block *bodyBlock = doStmtEval.lexicalSuccessor->block;
+      mlir::Block *exitBlock = doStmtEval.parentConstruct->constructExit->block;
+      IncrementLoopNestInfo &incrementLoopNestInfo = loopInfos.emplace_back();
+      const Fortran::parser::ScalarLogicalExpr *whileCondition = nullptr;
+      bool infiniteLoop = !loopControl.has_value();
+      if (infiniteLoop) {
+        assert(unstructuredContext && "infinite loop must be unstructured");
+        startBlock(headerBlock);
+      } else if ((whileCondition =
+                      std::get_if<Fortran::parser::ScalarLogicalExpr>(
+                          &loopControl->u))) {
+        assert(unstructuredContext && "while loop must be unstructured");
+        maybeStartBlock(preheaderBlock); // no block or empty block
+        startBlock(headerBlock);
+        genConditionalBranch(*whileCondition, bodyBlock, exitBlock);
+      } else if (const auto *bounds =
+                     std::get_if<Fortran::parser::LoopControl::Bounds>(
+                         &loopControl->u)) {
+        // Non-concurrent increment loop.
+        IncrementLoopInfo &info = incrementLoopNestInfo.emplace_back(
+            *bounds->name.thing.symbol, bounds->lower, bounds->upper,
+            bounds->step);
+        if (unstructuredContext) {
+          maybeStartBlock(preheaderBlock);
+          info.hasRealControl = info.loopVariableSym->GetType()->IsNumeric(
+              Fortran::common::TypeCategory::Real);
+          info.headerBlock = headerBlock;
+          info.bodyBlock = bodyBlock;
+          info.exitBlock = exitBlock;
+        }
+      } else {
+        llvm_unreachable("Cannot permute DO CONCURRENT");
+      }
+
+      // Increment loop begin code. (Infinite/while code was already generated.)
+      if (!infiniteLoop && !whileCondition)
+        genFIRIncrementLoopBegin(incrementLoopNestInfo, doStmtEval.dirs);
+    };
+
+    auto leaveLoop = [&](Fortran::lower::pft::Evaluation &eval,
+                         mlir::Block *headerBlock,
+                         IncrementLoopNestInfo &incrementLoopNestInfo) {
+      bool unstructuredContext = eval.lowerAsUnstructured();
+
+      Fortran::lower::pft::Evaluation &doStmtEval =
+          eval.getFirstNestedEvaluation();
+      auto *doStmt = doStmtEval.getIf<Fortran::parser::NonLabelDoStmt>();
+
+      const auto &loopControl =
+          std::get<std::optional<Fortran::parser::LoopControl>>(doStmt->t);
+      bool infiniteLoop = !loopControl.has_value();
+      const Fortran::parser::ScalarLogicalExpr *whileCondition =
+          std::get_if<Fortran::parser::ScalarLogicalExpr>(&loopControl->u);
+
+      auto iter = std::prev(eval.getNestedEvaluations().end());
+
+      // An EndDoStmt in unstructured code may start a new block.
+      Fortran::lower::pft::Evaluation &endDoEval = *iter;
+      assert(endDoEval.getIf<Fortran::parser::EndDoStmt>() && "no enddo stmt");
+      if (unstructuredContext)
+        maybeStartBlock(endDoEval.block);
+
+      // Loop end code.
+      if (infiniteLoop || whileCondition)
+        genBranch(headerBlock);
+      else
+        genFIRIncrementLoopEnd(incrementLoopNestInfo);
+
+      // This call may generate a branch in some contexts.
+      genFIR(endDoEval, unstructuredContext);
+    };
+
+    for (auto l : doStmts)
+      enterLoop(*l);
+
+    // Loop body code.
+    bool innermostUnstructuredContext = innermostDo->lowerAsUnstructured();
+
+    auto iter = innermostDo->getNestedEvaluations().begin();
+    for (auto end = --innermostDo->getNestedEvaluations().end(); iter != end;
+         ++iter)
+      genFIR(*iter, innermostUnstructuredContext);
+
+    for (auto &&[l, headerBlock, li] :
+         llvm::zip_equal(doStmts, headerBlocks, loopInfos))
+      leaveLoop(*l, headerBlock, li);
+  }
+
   /// Generate FIR for a DO construct. There are six variants:
   ///  - unstructured infinite and while loops
   ///  - structured and unstructured increment loops

flang/lib/Lower/OpenMP/Decomposer.cpp

@@ -98,14 +98,39 @@ ConstructQueue buildConstructQueue(
   return decompose.output;
 }
 
+// from clang
+static bool isOpenMPLoopTransformationDirective(llvm::omp::Directive DKind) {
+  return DKind == llvm::omp::Directive::OMPD_tile ||
+         DKind == llvm::omp::Directive::OMPD_unroll ||
+         DKind == llvm::omp::Directive::OMPD_reverse ||
+         DKind == llvm::omp::Directive::OMPD_interchange ||
+         DKind == llvm::omp::Directive::OMPD_stripe;
+}
+
+llvm::iterator_range<ConstructQueue::const_iterator> getNonTransformQueue(
+    llvm::iterator_range<ConstructQueue::const_iterator> range) {
+  // remove trailing loop transformations
+  auto b = range.begin();
+  auto e = range.end();
+  while (e != b) {
+    auto e2 = e - 1;
+    if (!isOpenMPLoopTransformationDirective(e2->id))
+      break;
+    e = e2;
+  }
+
+  return llvm::make_range(b, e);
+}
+
 bool matchLeafSequence(ConstructQueue::const_iterator item,
                        const ConstructQueue &queue,
                        llvm::omp::Directive directive) {
   llvm::ArrayRef<llvm::omp::Directive> leafDirs =
       llvm::omp::getLeafConstructsOrSelf(directive);
 
-  for (auto [dir, leaf] :
-       llvm::zip_longest(leafDirs, llvm::make_range(item, queue.end()))) {
+  for (auto [dir, leaf] : llvm::zip_longest(
+           leafDirs,
+           getNonTransformQueue(llvm::make_range(item, queue.end())))) {
     if (!dir.has_value() || !leaf.has_value())
       return false;
 

flang/lib/Lower/OpenMP/Decomposer.h

@@ -57,6 +57,10 @@ bool isLastItemInQueue(ConstructQueue::const_iterator item,
 bool matchLeafSequence(ConstructQueue::const_iterator item,
                        const ConstructQueue &queue,
                        llvm::omp::Directive directive);
+
+llvm::iterator_range<ConstructQueue::const_iterator> getNonTransformQueue(
+    llvm::iterator_range<ConstructQueue::const_iterator> range);
+
 } // namespace Fortran::lower::omp
 
 #endif // FORTRAN_LOWER_OPENMP_DECOMPOSER_H