Used OtherPredicates to simplify the PatFrag and to optimize the patterns.

cdevadas · cdevadas · commit 6dee07552bea · 2024-07-23T08:25:21.000Z
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -2026,6 +2026,8 @@ def HasAddNoCarryInsts : Predicate<"Subtarget->hasAddNoCarry()">,
 
 def NotHasAddNoCarryInsts : Predicate<"!Subtarget->hasAddNoCarry()">;
 
+def HasXNACKEnabled : Predicate<"Subtarget->isXNACKEnabled()">;
+
 def Has16BitInsts : Predicate<"Subtarget->has16BitInsts()">,
   AssemblerPredicate<(all_of Feature16BitInsts)>;
 
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -867,19 +867,12 @@ def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">;
 def SMRDBufferSgprImm : ComplexPattern<iPTR, 2, "SelectSMRDBufferSgprImm">;
 
 class SMRDAlignedLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr), [{
-  // Ignore the alignment check if XNACK support is disabled.
-  if (!Subtarget->isXNACKEnabled())
-    return true;
-
   // Returns true if it is a single dword load or naturally aligned multi-dword load.
   LoadSDNode *Ld = cast<LoadSDNode>(N);
   unsigned Size = Ld->getMemoryVT().getStoreSize();
   return Size <= 4 || Ld->getAlign().value() >= Size;
 }]> {
   let GISelPredicateCode = [{
-  if (!Subtarget->isXNACKEnabled())
-    return true;
-
   auto &Ld = cast<GLoad>(MI);
   TypeSize Size = Ld.getMMO().getSize().getValue();
   return Size <= 4 || Ld.getMMO().getAlign().value() >= Size;
@@ -888,79 +881,59 @@ class SMRDAlignedLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr),
 
 def aligned_smrd_load : SMRDAlignedLoadPat<smrd_load>;
 
-multiclass SMRD_Pattern <string Instr, ValueType vt, bit immci = true> {
+multiclass SMRD_Patterns <string Instr, ValueType vt, PatFrag frag,
+                          bit immci = true, string suffix = ""> {
+  // 1. IMM offset
+  def : GCNPat <
+    (frag (SMRDImm i64:$sbase, i32:$offset)),
+    (vt (!cast<SM_Pseudo>(Instr#"_IMM"#suffix) $sbase, $offset, 0))>;
+
+  // 2. 32-bit IMM offset on CI
+  if immci then def : GCNPat <
+    (frag (SMRDImm32 i64:$sbase, i32:$offset)),
+    (vt (!cast<InstSI>(Instr#"_IMM_ci"#suffix) $sbase, $offset, 0))> {
+    let SubtargetPredicate = isGFX7Only;
+  }
+
+  // 3. SGPR offset
+  def : GCNPat <
+    (frag (SMRDSgpr i64:$sbase, i32:$soffset)),
+    (vt (!cast<SM_Pseudo>(Instr#"_SGPR"#suffix) $sbase, $soffset, 0))> {
+    let SubtargetPredicate = isNotGFX9Plus;
+  }
+  def : GCNPat <
+    (frag (SMRDSgpr i64:$sbase, i32:$soffset)),
+    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM"#suffix) $sbase, $soffset, 0, 0))> {
+    let SubtargetPredicate = isGFX9Plus;
+  }
 
-  let AddedComplexity = 101 in {
-    // 1. IMM offset
-    def : GCNPat <
-      (aligned_smrd_load (SMRDImm i64:$sbase, i32:$offset)),
-      (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))>;
-
-    // 2. 32-bit IMM offset on CI
-    if immci then def : GCNPat <
-      (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)),
-      (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0))> {
-      let SubtargetPredicate = isGFX7Only;
-    }
-
-    // 3. SGPR offset
-    def : GCNPat <
-      (aligned_smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
-      (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $soffset, 0))> {
-      let SubtargetPredicate = isNotGFX9Plus;
-    }
-    def : GCNPat <
-      (aligned_smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
-      (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, 0, 0))> {
-      let SubtargetPredicate = isGFX9Plus;
-    }
-
-    // 4. SGPR+IMM offset
-    def : GCNPat <
-      (aligned_smrd_load (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)),
-      (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, $offset, 0))> {
-      let SubtargetPredicate = isGFX9Plus;
-    }
-
-    // 5. No offset
-    def : GCNPat <
-      (vt (aligned_smrd_load (i64 SReg_64:$sbase))),
-      (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0))>;
+  // 4. SGPR+IMM offset
+  def : GCNPat <
+    (frag (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)),
+    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM"#suffix) $sbase, $soffset, $offset, 0))> {
+    let SubtargetPredicate = isGFX9Plus;
   }
 
-  // The constrained multi-dword load equivalents.
+  // 5. No offset
+  def : GCNPat <
+    (vt (frag (i64 SReg_64:$sbase))),
+    (vt (!cast<SM_Pseudo>(Instr#"_IMM"#suffix) i64:$sbase, 0, 0))>;
+}
+
+multiclass SMRD_Pattern <string Instr, ValueType vt, bit immci = true> {
+  // High priority when XNACK is enabled and the load was naturally aligned.
+  let OtherPredicates = [HasXNACKEnabled], AddedComplexity = 102 in
+    defm: SMRD_Patterns <Instr, vt, aligned_smrd_load, immci>;
+
+  // XNACK is enabled and the load wasn't naturally aligned. The constrained sload variant.
   if !gt(vt.Size, 32) then {
-    let AddedComplexity = 100 in {
-      // 1. IMM offset
-      def : GCNPat <
-        (smrd_load (SMRDImm i64:$sbase, i32:$offset)),
-        (vt (!cast<SM_Pseudo>(Instr#"_IMM_ec") $sbase, $offset, 0))>;
-
-      // 2. SGPR offset
-      def : GCNPat <
-        (smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
-        (vt (!cast<SM_Pseudo>(Instr#"_SGPR_ec") $sbase, $soffset, 0))> {
-        let SubtargetPredicate = isNotGFX9Plus;
-      }
-      def : GCNPat <
-        (smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
-        (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM_ec") $sbase, $soffset, 0, 0))> {
-        let SubtargetPredicate = isGFX9Plus;
-      }
-
-      // 3. SGPR+IMM offset
-      def : GCNPat <
-        (smrd_load (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)),
-        (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM_ec") $sbase, $soffset, $offset, 0))> {
-        let SubtargetPredicate = isGFX9Plus;
-      }
-
-      // 4. No offset
-      def : GCNPat <
-        (vt (smrd_load (i64 SReg_64:$sbase))),
-        (vt (!cast<SM_Pseudo>(Instr#"_IMM_ec") i64:$sbase, 0, 0))>;
-    }
+    let OtherPredicates = [HasXNACKEnabled], AddedComplexity = 101 in
+      defm: SMRD_Patterns <Instr, vt, smrd_load, /*immci=*/false, /*suffix=*/"_ec">;
   }
+
+  // XNACK is disabled.
+  let AddedComplexity = 100 in
+    defm: SMRD_Patterns <Instr, vt, smrd_load, immci>;
 }
 
 multiclass SMLoad_Pattern <string Instr, ValueType vt, bit immci = true> {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir
@@ -1102,8 +1102,8 @@ body: |
     ; GFX7: liveins: $sgpr0_sgpr1
     ; GFX7-NEXT: {{  $}}
     ; GFX7-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
-    ; GFX7-NEXT: [[S_LOAD_DWORD_IMM_ci:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM_ci [[COPY]], 255, 0 :: (load (s32), addrspace 4)
-    ; GFX7-NEXT: $sgpr0 = COPY [[S_LOAD_DWORD_IMM_ci]]
+    ; GFX7-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 255, 0 :: (load (s32), addrspace 4)
+    ; GFX7-NEXT: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
     ;
     ; GFX8-LABEL: name: load_constant_s32_from_4_gep_1020
     ; GFX8: liveins: $sgpr0_sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-smrd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-smrd.mir
@@ -20,13 +20,11 @@ regBankSelected: true
 # GCN: [[PTR:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
 
 # Immediate offset:
-# SI: S_LOAD_DWORD_IMM [[PTR]], 1, 0
-# CI: S_LOAD_DWORD_IMM_ci [[PTR]], 1, 0
+# SICI: S_LOAD_DWORD_IMM [[PTR]], 1, 0
 # VI:   S_LOAD_DWORD_IMM [[PTR]], 4, 0
 
 # Max immediate offset for SI
-# SI: S_LOAD_DWORD_IMM [[PTR]], 255, 0
-# CI: S_LOAD_DWORD_IMM_ci [[PTR]], 255, 0
+# SICI: S_LOAD_DWORD_IMM [[PTR]], 255, 0
 # VI:   S_LOAD_DWORD_IMM [[PTR]], 1020, 0
 
 # Immediate overflow for SI
@@ -198,8 +196,7 @@ body: |
 # GCN: %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 44
 
 # VI: %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 64, 0 :: (dereferenceable invariant load (s32), addrspace 4)
-# SI: %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 16, 0 :: (dereferenceable invariant load (s32), addrspace 4)
-# CI: %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM_ci %0, 16, 0 :: (dereferenceable invariant load (s32), addrspace 4)
+# SICI: %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 16, 0 :: (dereferenceable invariant load (s32), addrspace 4)
 
 ---
 
diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll
@@ -729,15 +729,14 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f32_off4_slc(<4 x i32> inre
 ;
 ; G_GFX7-LABEL: raw_buffer_atomic_max_rtn_f32_off4_slc:
 ; G_GFX7:       ; %bb.0: ; %main_body
-; G_GFX7-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
-; G_GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; G_GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xf
+; G_GFX7-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
 ; G_GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; G_GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; G_GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; G_GFX7-NEXT:    buffer_atomic_fmax v0, v1, s[4:7], 4 offen glc slc
+; G_GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; G_GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; G_GFX7-NEXT:    buffer_atomic_fmax v0, v1, s[0:3], 4 offen glc slc
 ; G_GFX7-NEXT:    s_mov_b32 s2, -1
 ; G_GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; G_GFX7-NEXT:    s_mov_b64 s[0:1], s[6:7]
 ; G_GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; G_GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; G_GFX7-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
@@ -658,15 +658,14 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f32_off4_slc(ptr addrsp
 ;
 ; G_GFX7-LABEL: raw_ptr_buffer_atomic_max_rtn_f32_off4_slc:
 ; G_GFX7:       ; %bb.0: ; %main_body
-; G_GFX7-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
-; G_GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; G_GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xf
+; G_GFX7-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
 ; G_GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; G_GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; G_GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; G_GFX7-NEXT:    buffer_atomic_fmax v0, v1, s[4:7], 4 offen glc slc
+; G_GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; G_GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; G_GFX7-NEXT:    buffer_atomic_fmax v0, v1, s[0:3], 4 offen glc slc
 ; G_GFX7-NEXT:    s_mov_b32 s2, -1
 ; G_GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; G_GFX7-NEXT:    s_mov_b64 s[0:1], s[6:7]
 ; G_GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; G_GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; G_GFX7-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
@@ -492,9 +492,7 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> inre
 ;
 ; G_GFX7-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
 ; G_GFX7:       ; %bb.0: ; %main_body
-; G_GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
-; G_GFX7-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0xf
-; G_GFX7-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; G_GFX7-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
 ; G_GFX7-NEXT:    s_mov_b32 m0, -1
 ; G_GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; G_GFX7-NEXT:    v_mov_b32_e32 v0, s4
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
@@ -492,9 +492,7 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
 ;
 ; G_GFX7-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
 ; G_GFX7:       ; %bb.0: ; %main_body
-; G_GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
-; G_GFX7-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0xf
-; G_GFX7-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; G_GFX7-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
 ; G_GFX7-NEXT:    s_mov_b32 m0, -1
 ; G_GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; G_GFX7-NEXT:    v_mov_b32_e32 v0, s4
