From 24a7257528653830637c2ac910b6933a82c2bc38 Mon Sep 17 00:00:00 2001 From: Ryotaro Kasuga Date: Tue, 15 Jul 2025 13:01:45 +0000 Subject: [PATCH 1/3] [LoopInterchange] Exit early when all loops have deps in all directions --- .../lib/Transforms/Scalar/LoopInterchange.cpp | 12 +++++ .../LoopInterchange/bail-out-all-deps.ll | 45 +++++++++++++++++++ .../LoopInterchange/confused-dependence.ll | 16 ++++--- .../legality-for-scalar-deps.ll | 10 ++--- .../loop-interchange-optimization-remarks.ll | 6 +-- .../LoopInterchange/unique-dep-matrix.ll | 3 +- 6 files changed, 76 insertions(+), 16 deletions(-) create mode 100644 llvm/test/Transforms/LoopInterchange/bail-out-all-deps.ll diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp index a5008907b9014..68ecf37f90628 100644 --- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp +++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp @@ -221,6 +221,18 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level, Dep.push_back('I'); } + // If there is a direction vector with all entries being '*', we cannot + // prove the legality of the interchange for arbitrary pairs of loops. + // Exit early in this case to save compile time. + if (all_of(Dep, [](char C) { return C == '*'; })) { + ORE->emit([&]() { + return OptimizationRemarkMissed(DEBUG_TYPE, "Dependence", + L->getStartLoc(), L->getHeader()) + << "All loops have dependencies in all directions."; + }); + return false; + } + // Make sure we only add unique entries to the dependency matrix. if (Seen.insert(StringRef(Dep.data(), Dep.size())).second) DepMatrix.push_back(Dep); diff --git a/llvm/test/Transforms/LoopInterchange/bail-out-all-deps.ll b/llvm/test/Transforms/LoopInterchange/bail-out-all-deps.ll new file mode 100644 index 0000000000000..e22d17e5f5400 --- /dev/null +++ b/llvm/test/Transforms/LoopInterchange/bail-out-all-deps.ll @@ -0,0 +1,45 @@ +; RUN: opt < %s -passes=loop-interchange -pass-remarks-output=%t \ +; RUN: -disable-output +; RUN: FileCheck -input-file %t %s + +; Check that loop interchange bail out early when all loops have dependencies +; in (potentially) all directions. +; +; for (int i = 0; i < 4; i++) +; for (int j = 0; j < 4; j++) +; A[i & val][j & val] = 42; + +; CHECK: --- !Missed +; CHECK-NEXT: Pass: loop-interchange +; CHECK-NEXT: Name: Dependence +; CHECK-NEXT: Function: f +; CHECK-NEXT: Args: +; CHECK-NEXT: - String: All loops have dependencies in all directions. +; CHECK-NEXT: ... +define void @f(ptr %A, i32 %val) { +entry: + br label %for.i.header + +for.i.header: + %i = phi i32 [ 0, %entry ], [ %i.next, %for.i.latch ] + %subscript.0 = and i32 %i, %val + %i2 = mul i32 %i, %i + br label %for.j + +for.j: + %j = phi i32 [ 0, %for.i.header ], [ %j.next, %for.j ] + %subscript.1 = and i32 %j, %val + %idx = getelementptr inbounds [4 x [4 x i32]], ptr %A, i32 0, i32 %subscript.0, i32 %subscript.1 + store i32 42, ptr %idx, align 4 + %j.next = add i32 %j, 1 + %j.exit = icmp eq i32 %j.next, 4 + br i1 %j.exit, label %for.i.latch, label %for.j + +for.i.latch: + %i.next = add i32 %i, 1 + %i.exit = icmp eq i32 %i.next, 4 + br i1 %i.exit, label %exit, label %for.i.header + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopInterchange/confused-dependence.ll b/llvm/test/Transforms/LoopInterchange/confused-dependence.ll index 49b7b0e4797b8..94080949f0af8 100644 --- a/llvm/test/Transforms/LoopInterchange/confused-dependence.ll +++ b/llvm/test/Transforms/LoopInterchange/confused-dependence.ll @@ -1,6 +1,6 @@ -; REQUIRES: asserts -; RUN: opt < %s -passes=loop-interchange -verify-dom-info -verify-loop-info \ -; RUN: -disable-output -debug 2>&1 | FileCheck %s +; RUN: opt < %s -passes=loop-interchange -pass-remarks-output=%t \ +; RUN: -disable-output +; RUN: FileCheck -input-file %t %s ;; In the following case, p0 and p1 may alias, so the direction vector must be [* *]. ;; @@ -10,9 +10,13 @@ ;; p0[4 * i + j] = p1[4 * j + i]; ;; } -; CHECK: Dependency matrix before interchange: -; CHECK-NEXT: * * -; CHECK-NEXT: Processing InnerLoopId = 1 and OuterLoopId = 0 +; CHECK: --- !Missed +; CHECK-NEXT: Pass: loop-interchange +; CHECK-NEXT: Name: Dependence +; CHECK-NEXT: Function: may_alias +; CHECK-NEXT: Args: +; CHECK-NEXT: - String: All loops have dependencies in all directions. +; CHECK-NEXT: ... define void @may_alias(ptr %p0, ptr %p1) { entry: br label %for.i.header diff --git a/llvm/test/Transforms/LoopInterchange/legality-for-scalar-deps.ll b/llvm/test/Transforms/LoopInterchange/legality-for-scalar-deps.ll index c30f9a399fed8..5f4a8486d9ad7 100644 --- a/llvm/test/Transforms/LoopInterchange/legality-for-scalar-deps.ll +++ b/llvm/test/Transforms/LoopInterchange/legality-for-scalar-deps.ll @@ -21,13 +21,13 @@ ; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: issue46867 ; CHECK-NEXT: Args: -; CHECK-NEXT: - String: Cannot interchange loops due to dependences. +; CHECK-NEXT: - String: All loops have dependencies in all directions. ; CHECK: --- !Missed ; CHECK-NEXT: Pass: loop-interchange ; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: issue46867 ; CHECK-NEXT: Args: -; CHECK-NEXT: - String: Cannot interchange loops due to dependences. +; CHECK-NEXT: - String: All loops have dependencies in all directions. define void @issue46867(ptr noundef captures(none) %s, i32 noundef %c, ptr noundef readonly captures(none) %ff) { entry: %tobool7.not = icmp eq i32 %c, 0 @@ -121,7 +121,7 @@ land.end: ; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: issue47401 ; CHECK-NEXT: Args: -; CHECK-NEXT: - String: Cannot interchange loops due to dependences. +; CHECK-NEXT: - String: All loops have dependencies in all directions. define void @issue47401(ptr noundef writeonly captures(none) %e, ptr noundef readonly captures(none) %bb) { entry: br label %for.cond1.preheader @@ -175,7 +175,7 @@ land.end: ; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: issue47295 ; CHECK-NEXT: Args: -; CHECK-NEXT: - String: Cannot interchange loops due to dependences. +; CHECK-NEXT: - String: All loops have dependencies in all directions. define void @issue47295(ptr noundef captures(none) %f, ptr noundef writeonly captures(none) %cc) { entry: br label %for.cond1.preheader @@ -221,7 +221,7 @@ for.body4: ; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: issue54176 ; CHECK-NEXT: Args: -; CHECK-NEXT: - String: Cannot interchange loops due to dependences. +; CHECK-NEXT: - String: All loops have dependencies in all directions. define void @issue54176(i32 noundef %n, i32 noundef %m, ptr noundef captures(none) %aa, ptr noundef readonly captures(none) %bb, ptr noundef writeonly captures(none) %cc) { entry: diff --git a/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll b/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll index 73a566a310157..14836ba73433d 100644 --- a/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll +++ b/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll @@ -71,7 +71,7 @@ for.end19: ; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: test01 ; CHECK-NEXT: Args: -; CHECK-NEXT: - String: Cannot interchange loops due to dependences. +; CHECK-NEXT: - String: All loops have dependencies in all directions. ; CHECK-NEXT: ... ; DELIN: --- !Analysis @@ -147,7 +147,7 @@ define void @test02(i32 %k, i32 %N) { ; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: test02 ; CHECK-NEXT: Args: -; CHECK-NEXT: - String: Cannot interchange loops due to dependences. +; CHECK-NEXT: - String: All loops have dependencies in all directions. ; CHECK-NEXT: ... ; DELIN: --- !Analysis @@ -290,7 +290,7 @@ for.end17: ; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: test04 ; CHECK-NEXT: Args: -; CHECK-NEXT: - String: Cannot interchange loops due to dependences. +; CHECK-NEXT: - String: All loops have dependencies in all directions. ; CHECK-NEXT: ... ; DELIN: --- !Missed diff --git a/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll b/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll index 68089b43121c5..3af9e7304e3be 100644 --- a/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll +++ b/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll @@ -2,14 +2,13 @@ ; RUN: opt < %s -passes=loop-interchange -S -debug 2>&1 | FileCheck %s ; CHECK: Dependency matrix before interchange: -; CHECK-NEXT: * * ; CHECK-NEXT: = * ; CHECK-NEXT: < * ; CHECK-NEXT: Processing InnerLoopId ; This example is taken from github issue #54176 ; -define void @foo(i32 noundef %n, i32 noundef %m, ptr nocapture noundef %aa, ptr nocapture noundef readonly %bb, ptr nocapture noundef writeonly %cc) { +define void @foo(i32 noundef %n, i32 noundef %m, ptr nocapture noundef noalias %aa, ptr nocapture noundef readonly noalias %bb, ptr nocapture noundef writeonly noalias %cc) { entry: %arrayidx7 = getelementptr inbounds i8, ptr %aa, i64 512 br label %for.cond1.preheader From c51edffc0609541624b04b9e832eab3df39123da Mon Sep 17 00:00:00 2001 From: Ryotaro Kasuga Date: Thu, 25 Sep 2025 07:29:12 +0000 Subject: [PATCH 2/3] fix test --- .../LoopInterchange/bail-out-all-deps.ll | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/llvm/test/Transforms/LoopInterchange/bail-out-all-deps.ll b/llvm/test/Transforms/LoopInterchange/bail-out-all-deps.ll index e22d17e5f5400..83cfd91c4da4c 100644 --- a/llvm/test/Transforms/LoopInterchange/bail-out-all-deps.ll +++ b/llvm/test/Transforms/LoopInterchange/bail-out-all-deps.ll @@ -2,12 +2,12 @@ ; RUN: -disable-output ; RUN: FileCheck -input-file %t %s -; Check that loop interchange bail out early when all loops have dependencies -; in (potentially) all directions. +; Check that loop interchange bails out early when finding a direction vector +; with all '*' elements. ; ; for (int i = 0; i < 4; i++) ; for (int j = 0; j < 4; j++) -; A[i & val][j & val] = 42; +; A[i & val][j & val] = 0; ; CHECK: --- !Missed ; CHECK-NEXT: Pass: loop-interchange @@ -16,29 +16,28 @@ ; CHECK-NEXT: Args: ; CHECK-NEXT: - String: All loops have dependencies in all directions. ; CHECK-NEXT: ... -define void @f(ptr %A, i32 %val) { +define void @f(ptr %A, i64 %val) { entry: br label %for.i.header for.i.header: - %i = phi i32 [ 0, %entry ], [ %i.next, %for.i.latch ] - %subscript.0 = and i32 %i, %val - %i2 = mul i32 %i, %i + %i = phi i64 [ 0, %entry ], [ %i.next, %for.i.latch ] br label %for.j for.j: - %j = phi i32 [ 0, %for.i.header ], [ %j.next, %for.j ] - %subscript.1 = and i32 %j, %val - %idx = getelementptr inbounds [4 x [4 x i32]], ptr %A, i32 0, i32 %subscript.0, i32 %subscript.1 - store i32 42, ptr %idx, align 4 - %j.next = add i32 %j, 1 - %j.exit = icmp eq i32 %j.next, 4 - br i1 %j.exit, label %for.i.latch, label %for.j + %j = phi i64 [ 0, %for.i.header ], [ %j.next, %for.j ] + %subscript.0 = and i64 %i, %val + %subscript.1 = and i64 %j, %val + %idx = getelementptr inbounds [4 x i8], ptr %A, i64 %subscript.0, i64 %subscript.1 + store i8 0, ptr %idx + %j.next = add nuw nsw i64 %j, 1 + %exit.j = icmp eq i64 %j.next, 4 + br i1 %exit.j, label %for.i.latch, label %for.j for.i.latch: - %i.next = add i32 %i, 1 - %i.exit = icmp eq i32 %i.next, 4 - br i1 %i.exit, label %exit, label %for.i.header + %i.next = add nuw nsw i64 %i, 1 + %exit.i = icmp eq i64 %i.next, 4 + br i1 %exit.i, label %exit, label %for.i.header exit: ret void From 0e4e759d9e25696e36d608720aae703c0690a701 Mon Sep 17 00:00:00 2001 From: Ryotaro Kasuga Date: Thu, 25 Sep 2025 09:04:36 +0000 Subject: [PATCH 3/3] adjust comment --- llvm/lib/Transforms/Scalar/LoopInterchange.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp index 574283c917392..28ae4f0a0aad9 100644 --- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp +++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp @@ -260,9 +260,8 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level, Dep.push_back('I'); } - // If there is a direction vector with all entries being '*', we cannot - // prove the legality of the interchange for arbitrary pairs of loops. - // Exit early in this case to save compile time. + // If all the elements of any direction vector have only '*', legality + // can't be proven. Exit early to save compile time. if (all_of(Dep, [](char C) { return C == '*'; })) { ORE->emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "Dependence",