From c65af8046ba9fd26cecbe11824e5430e0503090a Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 25 Apr 2025 17:22:47 -0700 Subject: [PATCH] [Targets] Migrate from atomic_load_8/16/32/64 to atomic_load_nonext_8/16/32/64. NFC atomic_load_8/16/32/64 will be removed in a separate patch as it will affect out of tree targets. --- .../lib/Target/AArch64/AArch64InstrAtomics.td | 84 ++++++++++--------- llvm/lib/Target/AMDGPU/AMDGPUInstructions.td | 6 +- llvm/lib/Target/AMDGPU/BUFInstructions.td | 6 +- llvm/lib/Target/AMDGPU/DSInstructions.td | 6 +- llvm/lib/Target/AMDGPU/FLATInstructions.td | 12 +-- llvm/lib/Target/AMDGPU/SIInstrInfo.td | 30 ++++--- llvm/lib/Target/ARM/ARMInstrInfo.td | 8 +- llvm/lib/Target/ARM/ARMInstrThumb.td | 4 +- llvm/lib/Target/ARM/ARMInstrThumb2.td | 8 +- llvm/lib/Target/AVR/AVRInstrInfo.td | 4 +- llvm/lib/Target/BPF/BPFInstrInfo.td | 8 +- llvm/lib/Target/Hexagon/HexagonPatterns.td | 12 +-- .../Target/LoongArch/LoongArchInstrInfo.td | 4 +- llvm/lib/Target/Mips/MicroMipsInstrInfo.td | 2 +- llvm/lib/Target/Mips/Mips64InstrInfo.td | 2 +- llvm/lib/Target/Mips/MipsInstrInfo.td | 2 +- llvm/lib/Target/PowerPC/PPCInstr64Bit.td | 4 +- llvm/lib/Target/PowerPC/PPCInstrInfo.td | 4 +- llvm/lib/Target/PowerPC/PPCInstrP10.td | 8 +- llvm/lib/Target/RISCV/RISCVGISel.td | 4 +- llvm/lib/Target/Sparc/SparcInstr64Bit.td | 6 +- llvm/lib/Target/Sparc/SparcInstrInfo.td | 4 +- llvm/lib/Target/VE/VEInstrInfo.td | 8 +- .../WebAssembly/WebAssemblyInstrAtomics.td | 10 +-- llvm/lib/Target/X86/X86InstrCompiler.td | 80 +++++++++--------- 25 files changed, 168 insertions(+), 158 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td index 28d45fe25d30c..f3734e05ae667 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -55,9 +55,9 @@ let Predicates = [HasRCPC] in { // 16-bit loads def : Pat<(acquiring_load GPR64sp:$ptr), (LDAPRH GPR64sp:$ptr)>; // 32-bit loads - def : Pat<(acquiring_load GPR64sp:$ptr), (LDAPRW GPR64sp:$ptr)>; + def : Pat<(acquiring_load GPR64sp:$ptr), (LDAPRW GPR64sp:$ptr)>; // 64-bit loads - def : Pat<(acquiring_load GPR64sp:$ptr), (LDAPRX GPR64sp:$ptr)>; + def : Pat<(acquiring_load GPR64sp:$ptr), (LDAPRX GPR64sp:$ptr)>; } // 8-bit loads @@ -93,62 +93,66 @@ def : Pat<(relaxed_load (LDURHHi GPR64sp:$Rn, simm9:$offset)>; // 32-bit loads -def : Pat<(seq_cst_load GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>; -def : Pat<(acquiring_load GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>; -def : Pat<(relaxed_load (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, - ro_Wextend32:$extend)), +def : Pat<(seq_cst_load GPR64sp:$ptr), + (LDARW GPR64sp:$ptr)>; +def : Pat<(acquiring_load GPR64sp:$ptr), + (LDARW GPR64sp:$ptr)>; +def : Pat<(relaxed_load + (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)), (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; -def : Pat<(relaxed_load (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, - ro_Xextend32:$extend)), +def : Pat<(relaxed_load + (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)), (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; -def : Pat<(relaxed_load (am_indexed32 GPR64sp:$Rn, - uimm12s4:$offset)), +def : Pat<(relaxed_load + (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset)), (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>; -def : Pat<(relaxed_load +def : Pat<(relaxed_load (am_unscaled32 GPR64sp:$Rn, simm9:$offset)), (LDURWi GPR64sp:$Rn, simm9:$offset)>; // 64-bit loads -def : Pat<(seq_cst_load GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>; -def : Pat<(acquiring_load GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>; -def : Pat<(relaxed_load (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, - ro_Wextend64:$extend)), +def : Pat<(seq_cst_load GPR64sp:$ptr), + (LDARX GPR64sp:$ptr)>; +def : Pat<(acquiring_load GPR64sp:$ptr), + (LDARX GPR64sp:$ptr)>; +def : Pat<(relaxed_load + (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)), (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; -def : Pat<(relaxed_load (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, - ro_Xextend64:$extend)), +def : Pat<(relaxed_load + (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)), (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; -def : Pat<(relaxed_load (am_indexed64 GPR64sp:$Rn, - uimm12s8:$offset)), +def : Pat<(relaxed_load + (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)), (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>; -def : Pat<(relaxed_load +def : Pat<(relaxed_load (am_unscaled64 GPR64sp:$Rn, simm9:$offset)), (LDURXi GPR64sp:$Rn, simm9:$offset)>; // FP 32-bit loads -def : Pat<(f32 (bitconvert (i32 (relaxed_load (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, - ro_Wextend32:$extend))))), +def : Pat<(f32 (bitconvert (i32 (relaxed_load + (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend))))), (LDRSroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; -def : Pat<(f32 (bitconvert (i32 (relaxed_load (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, - ro_Xextend32:$extend))))), +def : Pat<(f32 (bitconvert (i32 (relaxed_load + (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend))))), (LDRSroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; -def : Pat<(f32 (bitconvert (i32 (relaxed_load (am_indexed32 GPR64sp:$Rn, - uimm12s8:$offset))))), +def : Pat<(f32 (bitconvert (i32 (relaxed_load + (am_indexed32 GPR64sp:$Rn, uimm12s8:$offset))))), (LDRSui GPR64sp:$Rn, uimm12s8:$offset)>; -def : Pat<(f32 (bitconvert (i32 (relaxed_load +def : Pat<(f32 (bitconvert (i32 (relaxed_load (am_unscaled32 GPR64sp:$Rn, simm9:$offset))))), (LDURSi GPR64sp:$Rn, simm9:$offset)>; // FP 64-bit loads -def : Pat<(f64 (bitconvert (i64 (relaxed_load (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, - ro_Wextend64:$extend))))), +def : Pat<(f64 (bitconvert (i64 (relaxed_load + (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend))))), (LDRDroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; -def : Pat<(f64 (bitconvert (i64 (relaxed_load (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, - ro_Xextend64:$extend))))), +def : Pat<(f64 (bitconvert (i64 (relaxed_load + (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend))))), (LDRDroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; -def : Pat<(f64 (bitconvert (i64 (relaxed_load (am_indexed64 GPR64sp:$Rn, - uimm12s8:$offset))))), +def : Pat<(f64 (bitconvert (i64 (relaxed_load + (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))))), (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>; -def : Pat<(f64 (bitconvert (i64 (relaxed_load +def : Pat<(f64 (bitconvert (i64 (relaxed_load (am_unscaled64 GPR64sp:$Rn, simm9:$offset))))), (LDURDi GPR64sp:$Rn, simm9:$offset)>; @@ -561,16 +565,16 @@ let Predicates = [HasLSFE] in { let Predicates = [HasRCPC3, HasNEON] in { // LDAP1 loads def : Pat<(vector_insert (v2i64 VecListOne128:$Rd), - (i64 (acquiring_load GPR64sp:$Rn)), (i64 VectorIndexD:$idx)), + (i64 (acquiring_load GPR64sp:$Rn)), (i64 VectorIndexD:$idx)), (LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>; def : Pat<(vector_insert (v2f64 VecListOne128:$Rd), - (f64 (bitconvert (i64 (acquiring_load GPR64sp:$Rn)))), (i64 VectorIndexD:$idx)), + (f64 (bitconvert (i64 (acquiring_load GPR64sp:$Rn)))), (i64 VectorIndexD:$idx)), (LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>; def : Pat<(v1i64 (scalar_to_vector - (i64 (acquiring_load GPR64sp:$Rn)))), + (i64 (acquiring_load GPR64sp:$Rn)))), (EXTRACT_SUBREG (LDAP1 (v2i64 (IMPLICIT_DEF)), (i64 0), GPR64sp:$Rn), dsub)>; def : Pat<(v1f64 (scalar_to_vector - (f64 (bitconvert (i64 (acquiring_load GPR64sp:$Rn)))))), + (f64 (bitconvert (i64 (acquiring_load GPR64sp:$Rn)))))), (EXTRACT_SUBREG (LDAP1 (v2f64 (IMPLICIT_DEF)), (i64 0), GPR64sp:$Rn), dsub)>; // STL1 stores @@ -597,10 +601,10 @@ let Predicates = [HasRCPC_IMMO, UseLDAPUR] in { def : Pat<(acquiring_load (am_unscaled16 GPR64sp:$Rn, simm9:$offset)), (LDAPURHi GPR64sp:$Rn, simm9:$offset)>; - def : Pat<(acquiring_load + def : Pat<(acquiring_load (am_unscaled32 GPR64sp:$Rn, simm9:$offset)), (LDAPURi GPR64sp:$Rn, simm9:$offset)>; - def : Pat<(acquiring_load + def : Pat<(acquiring_load (am_unscaled64 GPR64sp:$Rn, simm9:$offset)), (LDAPURXi GPR64sp:$Rn, simm9:$offset)>; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td index 6cc76b44f1e14..78a92d85cfd8e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -502,15 +502,15 @@ def zextloadi16_#as : PatFrag<(ops node:$ptr), (zextloadi16 node:$ptr)> { let IsLoad = 1; } -def atomic_load_16_#as : PatFrag<(ops node:$ptr), (atomic_load_16 node:$ptr)> { +def atomic_load_nonext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_16 node:$ptr)> { let IsAtomic = 1; } -def atomic_load_32_#as : PatFrag<(ops node:$ptr), (atomic_load_32 node:$ptr)> { +def atomic_load_nonext_32_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_32 node:$ptr)> { let IsAtomic = 1; } -def atomic_load_64_#as : PatFrag<(ops node:$ptr), (atomic_load_64 node:$ptr)> { +def atomic_load_nonext_64_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_64 node:$ptr)> { let IsAtomic = 1; } diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 7d64a3dd240c8..efcc81716a0f1 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -959,7 +959,7 @@ defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_aext_16_glo defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_zext_16_global>; defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_aext_8_global>; defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_zext_8_global>; -defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; +defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_nonext_16_global>; defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; @@ -1933,8 +1933,8 @@ def : MUBUFLoad_PatternADDR64 ; def : MUBUFLoad_PatternADDR64 ; -defm : MUBUFLoad_Atomic_Pattern ; -defm : MUBUFLoad_Atomic_Pattern ; +defm : MUBUFLoad_Atomic_Pattern ; +defm : MUBUFLoad_Atomic_Pattern ; } // End SubtargetPredicate = isGFX6GFX7 multiclass MUBUFLoad_PatternOffset_Common ; defm : DSReadPat_mc ; defm : DSReadPat_t16 ; defm : DSReadPat_mc ; -defm : DSReadPat_t16 ; +defm : DSReadPat_t16 ; defm : DSReadPat_mc ; defm : DSReadPat_mc ; defm : DSReadPat_mc ; -defm : DSReadPat_mc ; -defm : DSReadPat_mc ; +defm : DSReadPat_mc ; +defm : DSReadPat_mc ; let OtherPredicates = [D16PreservesUnusedBits] in { // TODO: Atomic loads diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index d8bb6e4378924..c17fda1346115 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -1541,7 +1541,7 @@ def : FlatLoadPat ; def : FlatLoadPat ; def : FlatLoadPat ; def : FlatLoadPat ; -def : FlatLoadPat ; +def : FlatLoadPat ; def : FlatLoadPat ; def : FlatLoadPat ; def : FlatLoadPat ; @@ -1573,8 +1573,8 @@ let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predi def : FlatStorePat ; } // End let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predicate = UseRealTrue16Insts -def : FlatLoadPat ; -def : FlatLoadPat ; +def : FlatLoadPat ; +def : FlatLoadPat ; def : FlatStorePat ; def : FlatStorePat ; @@ -1682,7 +1682,7 @@ defm : GlobalFLATLoadPats ; defm : GlobalFLATLoadPats ; defm : GlobalFLATLoadPats ; defm : GlobalFLATLoadPats ; -defm : GlobalFLATLoadPats ; +defm : GlobalFLATLoadPats ; defm : GlobalFLATLoadPats ; defm : GlobalFLATLoadPats ; defm : GlobalFLATLoadPats ; @@ -1733,8 +1733,8 @@ defm : GlobalFLATStorePats ; // There is no distinction for atomic load lowering during selection; // the memory legalizer will set the cache bits and insert the // appropriate waits. -defm : GlobalFLATLoadPats ; -defm : GlobalFLATLoadPats ; +defm : GlobalFLATLoadPats ; +defm : GlobalFLATLoadPats ; defm : GlobalFLATStorePats ; defm : GlobalFLATStorePats ; diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index ec1fd6fb60d57..5d837d853ac98 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -361,6 +361,12 @@ def load_glue : PatFrag <(ops node:$ptr), (unindexedload_glue node:$ptr)> { let IsNonExtLoad = 1; } +def atomic_load_nonext_glue : + PatFrag<(ops node:$ptr), (AMDGPUatomic_ld_glue node:$ptr)> { + let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic? + let IsNonExtLoad = true; +} + def atomic_load_zext_glue : PatFrag<(ops node:$ptr), (AMDGPUatomic_ld_glue node:$ptr)> { let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic? @@ -379,20 +385,20 @@ def atomic_load_aext_glue : let IsAnyExtLoad = true; } -def atomic_load_16_glue : PatFrag<(ops node:$ptr), - (AMDGPUatomic_ld_glue node:$ptr)> { +def atomic_load_nonext_16_glue : PatFrag<(ops node:$ptr), + (atomic_load_nonext_glue node:$ptr)> { let IsAtomic = 1; let MemoryVT = i16; } -def atomic_load_32_glue : PatFrag<(ops node:$ptr), - (AMDGPUatomic_ld_glue node:$ptr)> { +def atomic_load_nonext_32_glue : PatFrag<(ops node:$ptr), + (atomic_load_nonext_glue node:$ptr)> { let IsAtomic = 1; let MemoryVT = i32; } -def atomic_load_64_glue : PatFrag<(ops node:$ptr), - (AMDGPUatomic_ld_glue node:$ptr)> { +def atomic_load_nonext_64_glue : PatFrag<(ops node:$ptr), + (atomic_load_nonext_glue node:$ptr)> { let IsAtomic = 1; let MemoryVT = i64; } @@ -506,12 +512,12 @@ def load_align16_local_m0 : PatFrag<(ops node:$ptr), } let IsAtomic = 1, AddressSpaces = LoadAddress_local.AddrSpaces in { -def atomic_load_16_local_m0 : PatFrag<(ops node:$ptr), - (atomic_load_16_glue node:$ptr)>; -def atomic_load_32_local_m0 : PatFrag<(ops node:$ptr), - (atomic_load_32_glue node:$ptr)>; -def atomic_load_64_local_m0 : PatFrag<(ops node:$ptr), - (atomic_load_64_glue node:$ptr)>; +def atomic_load_nonext_16_local_m0 : PatFrag<(ops node:$ptr), + (atomic_load_nonext_16_glue node:$ptr)>; +def atomic_load_nonext_32_local_m0 : PatFrag<(ops node:$ptr), + (atomic_load_nonext_32_glue node:$ptr)>; +def atomic_load_nonext_64_local_m0 : PatFrag<(ops node:$ptr), + (atomic_load_nonext_64_glue node:$ptr)>; def atomic_load_zext_8_local_m0 : PatFrag<(ops node:$ptr), (atomic_load_zext_8_glue node:$ptr)>; diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td index 1ce9190a68f3c..c682f597401ec 100644 --- a/llvm/lib/Target/ARM/ARMInstrInfo.td +++ b/llvm/lib/Target/ARM/ARMInstrInfo.td @@ -5384,7 +5384,7 @@ class acquiring_load def atomic_load_azext_acquire_8 : acquiring_load; def atomic_load_azext_acquire_16 : acquiring_load; -def atomic_load_acquire_32 : acquiring_load; +def atomic_load_nonext_acquire_32 : acquiring_load; class releasing_store : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr), [{ @@ -5399,7 +5399,7 @@ def atomic_store_release_32 : releasing_store; let AddedComplexity = 8 in { def : ARMPat<(atomic_load_azext_acquire_8 addr_offset_none:$addr), (LDAB addr_offset_none:$addr)>; def : ARMPat<(atomic_load_azext_acquire_16 addr_offset_none:$addr), (LDAH addr_offset_none:$addr)>; - def : ARMPat<(atomic_load_acquire_32 addr_offset_none:$addr), (LDA addr_offset_none:$addr)>; + def : ARMPat<(atomic_load_nonext_acquire_32 addr_offset_none:$addr), (LDA addr_offset_none:$addr)>; def : ARMPat<(atomic_store_release_8 addr_offset_none:$addr, GPR:$val), (STLB GPR:$val, addr_offset_none:$addr)>; def : ARMPat<(atomic_store_release_16 addr_offset_none:$addr, GPR:$val), (STLH GPR:$val, addr_offset_none:$addr)>; def : ARMPat<(atomic_store_release_32 addr_offset_none:$addr, GPR:$val), (STL GPR:$val, addr_offset_none:$addr)>; @@ -6220,9 +6220,9 @@ def : ARMPat<(atomic_load_azext_8 addrmode_imm12:$src), (LDRBi12 addrmode_imm12:$src)>; def : ARMPat<(atomic_load_azext_16 addrmode3:$src), (LDRH addrmode3:$src)>; -def : ARMPat<(atomic_load_32 ldst_so_reg:$src), +def : ARMPat<(atomic_load_nonext_32 ldst_so_reg:$src), (LDRrs ldst_so_reg:$src)>; -def : ARMPat<(atomic_load_32 addrmode_imm12:$src), +def : ARMPat<(atomic_load_nonext_32 addrmode_imm12:$src), (LDRi12 addrmode_imm12:$src)>; def : ARMPat<(atomic_store_8 GPR:$val, ldst_so_reg:$ptr), (STRBrs GPR:$val, ldst_so_reg:$ptr)>; diff --git a/llvm/lib/Target/ARM/ARMInstrThumb.td b/llvm/lib/Target/ARM/ARMInstrThumb.td index feda22c89e925..e38cafdf55c46 100644 --- a/llvm/lib/Target/ARM/ARMInstrThumb.td +++ b/llvm/lib/Target/ARM/ARMInstrThumb.td @@ -1705,9 +1705,9 @@ def : T1Pat<(atomic_load_azext_16 t_addrmode_is2:$src), (tLDRHi t_addrmode_is2:$src)>; def : T1Pat<(atomic_load_azext_16 t_addrmode_rr:$src), (tLDRHr t_addrmode_rr:$src)>; -def : T1Pat<(atomic_load_32 t_addrmode_is4:$src), +def : T1Pat<(atomic_load_nonext_32 t_addrmode_is4:$src), (tLDRi t_addrmode_is4:$src)>; -def : T1Pat<(atomic_load_32 t_addrmode_rr:$src), +def : T1Pat<(atomic_load_nonext_32 t_addrmode_rr:$src), (tLDRr t_addrmode_rr:$src)>; def : T1Pat<(atomic_store_8 tGPR:$val, t_addrmode_is1:$ptr), (tSTRBi tGPR:$val, t_addrmode_is1:$ptr)>; diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td index f9a873a9483de..8f56fb0938dd0 100644 --- a/llvm/lib/Target/ARM/ARMInstrThumb2.td +++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td @@ -4911,11 +4911,11 @@ def : T2Pat<(atomic_load_azext_16 t2addrmode_negimm8:$addr), (t2LDRHi8 t2addrmode_negimm8:$addr)>; def : T2Pat<(atomic_load_azext_16 t2addrmode_so_reg:$addr), (t2LDRHs t2addrmode_so_reg:$addr)>; -def : T2Pat<(atomic_load_32 t2addrmode_imm12:$addr), +def : T2Pat<(atomic_load_nonext_32 t2addrmode_imm12:$addr), (t2LDRi12 t2addrmode_imm12:$addr)>; -def : T2Pat<(atomic_load_32 t2addrmode_negimm8:$addr), +def : T2Pat<(atomic_load_nonext_32 t2addrmode_negimm8:$addr), (t2LDRi8 t2addrmode_negimm8:$addr)>; -def : T2Pat<(atomic_load_32 t2addrmode_so_reg:$addr), +def : T2Pat<(atomic_load_nonext_32 t2addrmode_so_reg:$addr), (t2LDRs t2addrmode_so_reg:$addr)>; def : T2Pat<(atomic_store_8 GPR:$val, t2addrmode_imm12:$addr), (t2STRBi12 GPR:$val, t2addrmode_imm12:$addr)>; @@ -4939,7 +4939,7 @@ def : T2Pat<(atomic_store_32 GPR:$val, t2addrmode_so_reg:$addr), let AddedComplexity = 8, Predicates = [IsThumb, HasAcquireRelease, HasV7Clrex] in { def : Pat<(atomic_load_azext_acquire_8 addr_offset_none:$addr), (t2LDAB addr_offset_none:$addr)>; def : Pat<(atomic_load_azext_acquire_16 addr_offset_none:$addr), (t2LDAH addr_offset_none:$addr)>; - def : Pat<(atomic_load_acquire_32 addr_offset_none:$addr), (t2LDA addr_offset_none:$addr)>; + def : Pat<(atomic_load_nonext_acquire_32 addr_offset_none:$addr), (t2LDA addr_offset_none:$addr)>; def : Pat<(atomic_store_release_8 addr_offset_none:$addr, GPR:$val), (t2STLB GPR:$val, addr_offset_none:$addr)>; def : Pat<(atomic_store_release_16 addr_offset_none:$addr, GPR:$val), (t2STLH GPR:$val, addr_offset_none:$addr)>; def : Pat<(atomic_store_release_32 addr_offset_none:$addr, GPR:$val), (t2STL GPR:$val, addr_offset_none:$addr)>; diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.td b/llvm/lib/Target/AVR/AVRInstrInfo.td index da5e20e4e2859..606ce50f5f19e 100644 --- a/llvm/lib/Target/AVR/AVRInstrInfo.td +++ b/llvm/lib/Target/AVR/AVRInstrInfo.td @@ -987,8 +987,8 @@ class AtomicLoadOp // 16-bit operations use 16-bit load/store postincrement instructions, // which require PTRDISPREGS. -def AtomicLoad8 : AtomicLoad; -def AtomicLoad16 : AtomicLoad; +def AtomicLoad8 : AtomicLoad; +def AtomicLoad16 : AtomicLoad; def AtomicStore8 : AtomicStore; def AtomicStore16 : AtomicStore; diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index e717ac1a1d209..b21f1a0eee3b0 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -698,8 +698,8 @@ class acquiring_load let Predicates = [BPFHasLoadAcqStoreRel] in { def LDDACQ : LOAD_ACQUIREi64; - foreach P = [[relaxed_load, LDD], - [acquiring_load, LDDACQ], + foreach P = [[relaxed_load, LDD], + [acquiring_load, LDDACQ], ] in { def : Pat<(P[0] ADDRri:$addr), (P[1] ADDRri:$addr)>; } @@ -1341,10 +1341,10 @@ let Predicates = [BPFHasALU32] in { (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>; let Predicates = [BPFHasLoadAcqStoreRel] in { - foreach P = [[relaxed_load, LDW32], + foreach P = [[relaxed_load, LDW32], [relaxed_load, LDH32], [relaxed_load, LDB32], - [acquiring_load, LDWACQ32], + [acquiring_load, LDWACQ32], [acquiring_load, LDHACQ32], [acquiring_load, LDBACQ32], ] in { diff --git a/llvm/lib/Target/Hexagon/HexagonPatterns.td b/llvm/lib/Target/Hexagon/HexagonPatterns.td index 1be16c1739512..dd2a5a34afcc0 100644 --- a/llvm/lib/Target/Hexagon/HexagonPatterns.td +++ b/llvm/lib/Target/Hexagon/HexagonPatterns.td @@ -2258,8 +2258,8 @@ let AddedComplexity = 20 in { defm: Loadxi_pat; defm: Loadxi_pat; - defm: Loadxi_pat; - defm: Loadxi_pat; + defm: Loadxi_pat; + defm: Loadxi_pat; } let AddedComplexity = 30 in { @@ -2420,8 +2420,8 @@ let AddedComplexity = 60 in { def: Loada_pat; def: Loada_pat; - def: Loada_pat; - def: Loada_pat; + def: Loada_pat; + def: Loada_pat; } let AddedComplexity = 30 in { @@ -2465,8 +2465,8 @@ let AddedComplexity = 100 in { def: Loada_pat; def: Loada_pat; - def: Loada_pat; - def: Loada_pat; + def: Loada_pat; + def: Loada_pat; } let AddedComplexity = 70 in { diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td index b607dcb04149b..69d6266a4bf54 100644 --- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td @@ -1892,7 +1892,7 @@ def : Pat<(atomic_fence 7, timm), (DBAR 0b10000)>; // seqcst defm : LdPat; defm : LdPat; -defm : LdPat, Requires<[IsLA32]>; +defm : LdPat, Requires<[IsLA32]>; defm : LdPat, Requires<[IsLA64]>; class release_seqcst_store @@ -1934,7 +1934,7 @@ def PseudoAtomicStoreD def : Pat<(atomic_store_release_seqcst_64 GPR:$rj, GPR:$rk), (PseudoAtomicStoreD GPR:$rj, GPR:$rk)>; -defm : LdPat; +defm : LdPat; defm : StPat; defm : StPat; } // Predicates = [IsLA64] diff --git a/llvm/lib/Target/Mips/MicroMipsInstrInfo.td b/llvm/lib/Target/Mips/MicroMipsInstrInfo.td index 661c18e8c3952..b3fd8f422f429 100644 --- a/llvm/lib/Target/Mips/MicroMipsInstrInfo.td +++ b/llvm/lib/Target/Mips/MicroMipsInstrInfo.td @@ -1192,7 +1192,7 @@ def : WrapperPat, ISA_MICROMIPS; def : MipsPat<(atomic_load_asext_8 addr:$a), (LB_MM addr:$a)>, ISA_MICROMIPS; def : MipsPat<(atomic_load_asext_16 addr:$a), (LH_MM addr:$a)>, ISA_MICROMIPS; -def : MipsPat<(atomic_load_32 addr:$a), (LW_MM addr:$a)>, ISA_MICROMIPS; +def : MipsPat<(atomic_load_nonext_32 addr:$a), (LW_MM addr:$a)>, ISA_MICROMIPS; def : MipsPat<(i32 immLi16:$imm), (LI16_MM immLi16:$imm)>, ISA_MICROMIPS; diff --git a/llvm/lib/Target/Mips/Mips64InstrInfo.td b/llvm/lib/Target/Mips/Mips64InstrInfo.td index d028c95287a70..49e463cfd212c 100644 --- a/llvm/lib/Target/Mips/Mips64InstrInfo.td +++ b/llvm/lib/Target/Mips/Mips64InstrInfo.td @@ -897,7 +897,7 @@ def : MipsPat<(brcond (i32 (setne (and i32:$lhs, PowerOf2LO_i32:$mask), 0)), bb: def : MipsPat<(atomic_load_asext_8 addr:$a), (LB64 addr:$a)>, ISA_MIPS3, GPR_64; def : MipsPat<(atomic_load_asext_16 addr:$a), (LH64 addr:$a)>, ISA_MIPS3, GPR_64; def : MipsPat<(atomic_load_asext_32 addr:$a), (LW64 addr:$a)>, ISA_MIPS3, GPR_64; -def : MipsPat<(atomic_load_64 addr:$a), (LD addr:$a)>, ISA_MIPS3, GPR_64; +def : MipsPat<(atomic_load_nonext_64 addr:$a), (LD addr:$a)>, ISA_MIPS3, GPR_64; // Atomic store patterns. def : MipsPat<(atomic_store_8 GPR64:$v, addr:$a), (SB64 GPR64:$v, addr:$a)>, diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.td b/llvm/lib/Target/Mips/MipsInstrInfo.td index f17781dcab726..b6125b972717a 100644 --- a/llvm/lib/Target/Mips/MipsInstrInfo.td +++ b/llvm/lib/Target/Mips/MipsInstrInfo.td @@ -3360,7 +3360,7 @@ let AdditionalPredicates = [NotInMicroMips] in { // Atomic load patterns. def : MipsPat<(atomic_load_asext_8 addr:$a), (LB addr:$a)>, ISA_MIPS1; def : MipsPat<(atomic_load_asext_16 addr:$a), (LH addr:$a)>, ISA_MIPS1; - def : MipsPat<(atomic_load_32 addr:$a), (LW addr:$a)>, ISA_MIPS1; + def : MipsPat<(atomic_load_nonext_32 addr:$a), (LW addr:$a)>, ISA_MIPS1; // Atomic store patterns. def : MipsPat<(atomic_store_8 GPR32:$v, addr:$a), (SB GPR32:$v, addr:$a)>, diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td index 4205b3086a3c9..659c1a9079c33 100644 --- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td +++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td @@ -1958,8 +1958,8 @@ def : Pat<(PPCaddTls i64:$in, i64:$addr), (ADD8TLS $in, $addr)>; // 64-bits atomic loads and stores -def : Pat<(atomic_load_64 DSForm:$src), (LD memrix:$src)>; -def : Pat<(atomic_load_64 XForm:$src), (LDX memrr:$src)>; +def : Pat<(atomic_load_nonext_64 DSForm:$src), (LD memrix:$src)>; +def : Pat<(atomic_load_nonext_64 XForm:$src), (LDX memrr:$src)>; def : Pat<(atomic_store_64 i64:$val, DSForm:$ptr), (STD g8rc:$val, memrix:$ptr)>; def : Pat<(atomic_store_64 i64:$val, XForm:$ptr), (STDX g8rc:$val, memrr:$ptr)>; diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index cbf5d0188b79e..b70290df07b1c 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -5086,10 +5086,10 @@ defm : TrapExtendedMnemonic<"u", 31>; // Atomic loads def : Pat<(i32 (atomic_load_azext_8 DForm:$src)), (LBZ memri:$src)>; def : Pat<(i32 (atomic_load_azext_16 DForm:$src)), (LHZ memri:$src)>; -def : Pat<(i32 (atomic_load_32 DForm:$src)), (LWZ memri:$src)>; +def : Pat<(i32 (atomic_load_nonext_32 DForm:$src)), (LWZ memri:$src)>; def : Pat<(i32 (atomic_load_azext_8 XForm:$src)), (LBZX memrr:$src)>; def : Pat<(i32 (atomic_load_azext_16 XForm:$src)), (LHZX memrr:$src)>; -def : Pat<(i32 (atomic_load_32 XForm:$src)), (LWZX memrr:$src)>; +def : Pat<(i32 (atomic_load_nonext_32 XForm:$src)), (LWZX memrr:$src)>; // Atomic stores def : Pat<(atomic_store_8 i32:$val, DForm:$ptr), (STB gprc:$val, memri:$ptr)>; diff --git a/llvm/lib/Target/PowerPC/PPCInstrP10.td b/llvm/lib/Target/PowerPC/PPCInstrP10.td index 3f655d9738414..a7f758745efe2 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrP10.td +++ b/llvm/lib/Target/PowerPC/PPCInstrP10.td @@ -1279,9 +1279,9 @@ let Predicates = [PCRelativeMemops] in { (PLBZpc $ga, 0)>; def : Pat<(i32 (atomic_load_azext_16 (PPCmatpcreladdr PCRelForm:$ga))), (PLHZpc $ga, 0)>; - def : Pat<(i32 (atomic_load_32 (PPCmatpcreladdr PCRelForm:$ga))), + def : Pat<(i32 (atomic_load_nonext_32 (PPCmatpcreladdr PCRelForm:$ga))), (PLWZpc $ga, 0)>; - def : Pat<(i64 (atomic_load_64 (PPCmatpcreladdr PCRelForm:$ga))), + def : Pat<(i64 (atomic_load_nonext_64 (PPCmatpcreladdr PCRelForm:$ga))), (PLDpc $ga, 0)>; // Atomic Store @@ -2362,8 +2362,8 @@ let Predicates = [PrefixInstrs] in { // Atomic Load def : Pat<(i32 (atomic_load_azext_8 PDForm:$src)), (PLBZ memri34:$src)>; def : Pat<(i32 (atomic_load_azext_16 PDForm:$src)), (PLHZ memri34:$src)>; - def : Pat<(i32 (atomic_load_32 PDForm:$src)), (PLWZ memri34:$src)>; - def : Pat<(i64 (atomic_load_64 PDForm:$src)), (PLD memri34:$src)>; + def : Pat<(i32 (atomic_load_nonext_32 PDForm:$src)), (PLWZ memri34:$src)>; + def : Pat<(i64 (atomic_load_nonext_64 PDForm:$src)), (PLD memri34:$src)>; // Atomic Store def : Pat<(atomic_store_8 i32:$RS, PDForm:$dst), (PSTB $RS, memri34:$dst)>; diff --git a/llvm/lib/Target/RISCV/RISCVGISel.td b/llvm/lib/Target/RISCV/RISCVGISel.td index 36f26620655da..20ade6c39a8cf 100644 --- a/llvm/lib/Target/RISCV/RISCVGISel.td +++ b/llvm/lib/Target/RISCV/RISCVGISel.td @@ -110,14 +110,14 @@ def : StPat; let Predicates = [HasAtomicLdSt] in { def : LdPat; - def : LdPat; + def : LdPat; def : StPat; def : StPat; } let Predicates = [HasAtomicLdSt, IsRV64] in { - def : LdPat; + def : LdPat; def : StPat; } diff --git a/llvm/lib/Target/Sparc/SparcInstr64Bit.td b/llvm/lib/Target/Sparc/SparcInstr64Bit.td index 000612534e89d..372ab80a3bb71 100644 --- a/llvm/lib/Target/Sparc/SparcInstr64Bit.td +++ b/llvm/lib/Target/Sparc/SparcInstr64Bit.td @@ -472,9 +472,9 @@ let Predicates = [Is64Bit, HasV9], Constraints = "$swap = $rd" in { let Predicates = [Is64Bit] in { -// atomic_load_64 addr -> load addr -def : Pat<(i64 (atomic_load_64 ADDRrr:$src)), (LDXrr ADDRrr:$src)>; -def : Pat<(i64 (atomic_load_64 ADDRri:$src)), (LDXri ADDRri:$src)>; +// atomic_load_nonext_64 addr -> load addr +def : Pat<(i64 (atomic_load_nonext_64 ADDRrr:$src)), (LDXrr ADDRrr:$src)>; +def : Pat<(i64 (atomic_load_nonext_64 ADDRri:$src)), (LDXri ADDRri:$src)>; // atomic_store_64 val, addr -> store val, addr def : Pat<(atomic_store_64 i64:$val, ADDRrr:$dst), (STXrr ADDRrr:$dst, $val)>; diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.td b/llvm/lib/Target/Sparc/SparcInstrInfo.td index b867a1dab7e24..02f4b202e9645 100644 --- a/llvm/lib/Target/Sparc/SparcInstrInfo.td +++ b/llvm/lib/Target/Sparc/SparcInstrInfo.td @@ -1923,8 +1923,8 @@ def : Pat<(i32 (atomic_load_azext_8 ADDRrr:$src)), (LDUBrr ADDRrr:$src)>; def : Pat<(i32 (atomic_load_azext_8 ADDRri:$src)), (LDUBri ADDRri:$src)>; def : Pat<(i32 (atomic_load_azext_16 ADDRrr:$src)), (LDUHrr ADDRrr:$src)>; def : Pat<(i32 (atomic_load_azext_16 ADDRri:$src)), (LDUHri ADDRri:$src)>; -def : Pat<(i32 (atomic_load_32 ADDRrr:$src)), (LDrr ADDRrr:$src)>; -def : Pat<(i32 (atomic_load_32 ADDRri:$src)), (LDri ADDRri:$src)>; +def : Pat<(i32 (atomic_load_nonext_32 ADDRrr:$src)), (LDrr ADDRrr:$src)>; +def : Pat<(i32 (atomic_load_nonext_32 ADDRri:$src)), (LDri ADDRri:$src)>; // atomic_store val, addr -> store val, addr def : Pat<(atomic_store_8 i32:$val, ADDRrr:$dst), (STBrr ADDRrr:$dst, $val)>; diff --git a/llvm/lib/Target/VE/VEInstrInfo.td b/llvm/lib/Target/VE/VEInstrInfo.td index 6a6d3e069d218..7e3f29b3bd826 100644 --- a/llvm/lib/Target/VE/VEInstrInfo.td +++ b/llvm/lib/Target/VE/VEInstrInfo.td @@ -1796,8 +1796,8 @@ multiclass ATMLDm; defm : ATMLDm; -defm : ATMLDm; -defm : ATMLDm; +defm : ATMLDm; +defm : ATMLDm; // Optimized atomic loads with sext multiclass SXATMLDm; defm : SXATMLDm; -defm : SXATMLD32m; +defm : SXATMLD32m; // Optimized atomic loads with zext multiclass ZXATMLDm; defm : ZXATMLDm; -defm : ZXATMLD32m; +defm : ZXATMLD32m; // Atomic stores multiclass ATMSTm; defm ATOMIC_LOAD_I64 : AtomicLoad; // Select loads -defm : LoadPat; -defm : LoadPat; +defm : LoadPat; +defm : LoadPat; // Extending loads. Note that there are only zero-extending atomic loads, no // sign-extending loads. @@ -144,8 +144,8 @@ defm ATOMIC_LOAD32_U_I64 : AtomicLoad; // we match the patterns that the type legalizer expands them to. // Unlike regular loads, extension to i64 is handled differently than i32. -// i64 (zext (i8 (atomic_load_8))) gets legalized to -// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) +// i64 (zext (i8 (atomic_load_nonext_8))) gets legalized to +// i64 (and (i64 (anyext (i32 (atomic_load_zext_8)))), 255) // Extension to i32 is elided by SelectionDAG as our atomic loads are // zero-extending. def zext_aload_8_64 : @@ -156,7 +156,7 @@ def zext_aload_16_64 : (i64 (zext (i32 (atomic_load_azext_16 node:$addr))))>; def zext_aload_32_64 : PatFrag<(ops node:$addr), - (i64 (zext (i32 (atomic_load_32 node:$addr))))>; + (i64 (zext (i32 (atomic_load_nonext_32 node:$addr))))>; // We don't have single sext atomic load instructions. So for sext loads, we // match bare subword loads (for 32-bit results) and anyext loads (for 64-bit diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td index 9687ae29f1c78..167e27eddd71e 100644 --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -1083,27 +1083,27 @@ defm LXADD : ATOMIC_RMW_BINOP<0xc0, 0xc1, "xadd", "atomic_load_add">, TB, LOCK; * (see below the RELEASE_MOV* / ACQUIRE_MOV* pseudo-instructions) */ multiclass RELEASE_BINOP_MI { - def : Pat<(atomic_store_8 (op (atomic_load_8 addr:$dst), (i8 imm:$src)), + def : Pat<(atomic_store_8 (op (atomic_load_nonext_8 addr:$dst), (i8 imm:$src)), addr:$dst), (!cast(Name#"8mi") addr:$dst, imm:$src)>; - def : Pat<(atomic_store_16 (op (atomic_load_16 addr:$dst), (i16 imm:$src)), + def : Pat<(atomic_store_16 (op (atomic_load_nonext_16 addr:$dst), (i16 imm:$src)), addr:$dst), (!cast(Name#"16mi") addr:$dst, imm:$src)>; - def : Pat<(atomic_store_32 (op (atomic_load_32 addr:$dst), (i32 imm:$src)), + def : Pat<(atomic_store_32 (op (atomic_load_nonext_32 addr:$dst), (i32 imm:$src)), addr:$dst), (!cast(Name#"32mi") addr:$dst, imm:$src)>; - def : Pat<(atomic_store_64 (op (atomic_load_64 addr:$dst), (i64immSExt32:$src)), + def : Pat<(atomic_store_64 (op (atomic_load_nonext_64 addr:$dst), (i64immSExt32:$src)), addr:$dst), (!cast(Name#"64mi32") addr:$dst, (i64immSExt32:$src))>; - def : Pat<(atomic_store_8 (op (atomic_load_8 addr:$dst), (i8 GR8:$src)), addr:$dst), + def : Pat<(atomic_store_8 (op (atomic_load_nonext_8 addr:$dst), (i8 GR8:$src)), addr:$dst), (!cast(Name#"8mr") addr:$dst, GR8:$src)>; - def : Pat<(atomic_store_16 (op (atomic_load_16 addr:$dst), (i16 GR16:$src)), + def : Pat<(atomic_store_16 (op (atomic_load_nonext_16 addr:$dst), (i16 GR16:$src)), addr:$dst), (!cast(Name#"16mr") addr:$dst, GR16:$src)>; - def : Pat<(atomic_store_32 (op (atomic_load_32 addr:$dst), (i32 GR32:$src)), + def : Pat<(atomic_store_32 (op (atomic_load_nonext_32 addr:$dst), (i32 GR32:$src)), addr:$dst), (!cast(Name#"32mr") addr:$dst, GR32:$src)>; - def : Pat<(atomic_store_64 (op (atomic_load_64 addr:$dst), (i64 GR64:$src)), + def : Pat<(atomic_store_64 (op (atomic_load_nonext_64 addr:$dst), (i64 GR64:$src)), addr:$dst), (!cast(Name#"64mr") addr:$dst, GR64:$src)>; } @@ -1116,23 +1116,23 @@ defm : RELEASE_BINOP_MI<"SUB", sub>; // Atomic load + floating point patterns. // FIXME: This could also handle SIMD operations with *ps and *pd instructions. multiclass ATOMIC_LOAD_FP_BINOP_MI { - def : Pat<(op FR32:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))), + def : Pat<(op FR32:$src1, (bitconvert (i32 (atomic_load_nonext_32 addr:$src2)))), (!cast(Name#"SSrm") FR32:$src1, addr:$src2)>, Requires<[UseSSE1]>; - def : Pat<(op FR32:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))), + def : Pat<(op FR32:$src1, (bitconvert (i32 (atomic_load_nonext_32 addr:$src2)))), (!cast("V"#Name#"SSrm") FR32:$src1, addr:$src2)>, Requires<[UseAVX]>; - def : Pat<(op FR32X:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))), + def : Pat<(op FR32X:$src1, (bitconvert (i32 (atomic_load_nonext_32 addr:$src2)))), (!cast("V"#Name#"SSZrm") FR32X:$src1, addr:$src2)>, Requires<[HasAVX512]>; - def : Pat<(op FR64:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))), + def : Pat<(op FR64:$src1, (bitconvert (i64 (atomic_load_nonext_64 addr:$src2)))), (!cast(Name#"SDrm") FR64:$src1, addr:$src2)>, Requires<[UseSSE1]>; - def : Pat<(op FR64:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))), + def : Pat<(op FR64:$src1, (bitconvert (i64 (atomic_load_nonext_64 addr:$src2)))), (!cast("V"#Name#"SDrm") FR64:$src1, addr:$src2)>, Requires<[UseAVX]>; - def : Pat<(op FR64X:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))), + def : Pat<(op FR64X:$src1, (bitconvert (i64 (atomic_load_nonext_64 addr:$src2)))), (!cast("V"#Name#"SDZrm") FR64X:$src1, addr:$src2)>, Requires<[HasAVX512]>; } @@ -1155,27 +1155,27 @@ multiclass RELEASE_UNOP; + (add (atomic_load_nonext_8 addr:$dst), (i8 1)), + (add (atomic_load_nonext_16 addr:$dst), (i16 1)), + (add (atomic_load_nonext_32 addr:$dst), (i32 1)), + (add (atomic_load_nonext_64 addr:$dst), (i64 1))>; defm : RELEASE_UNOP<"DEC", - (add (atomic_load_8 addr:$dst), (i8 -1)), - (add (atomic_load_16 addr:$dst), (i16 -1)), - (add (atomic_load_32 addr:$dst), (i32 -1)), - (add (atomic_load_64 addr:$dst), (i64 -1))>; + (add (atomic_load_nonext_8 addr:$dst), (i8 -1)), + (add (atomic_load_nonext_16 addr:$dst), (i16 -1)), + (add (atomic_load_nonext_32 addr:$dst), (i32 -1)), + (add (atomic_load_nonext_64 addr:$dst), (i64 -1))>; } defm : RELEASE_UNOP<"NEG", - (ineg (i8 (atomic_load_8 addr:$dst))), - (ineg (i16 (atomic_load_16 addr:$dst))), - (ineg (i32 (atomic_load_32 addr:$dst))), - (ineg (i64 (atomic_load_64 addr:$dst)))>; + (ineg (i8 (atomic_load_nonext_8 addr:$dst))), + (ineg (i16 (atomic_load_nonext_16 addr:$dst))), + (ineg (i32 (atomic_load_nonext_32 addr:$dst))), + (ineg (i64 (atomic_load_nonext_64 addr:$dst)))>; defm : RELEASE_UNOP<"NOT", - (not (i8 (atomic_load_8 addr:$dst))), - (not (i16 (atomic_load_16 addr:$dst))), - (not (i32 (atomic_load_32 addr:$dst))), - (not (i64 (atomic_load_64 addr:$dst)))>; + (not (i8 (atomic_load_nonext_8 addr:$dst))), + (not (i16 (atomic_load_nonext_16 addr:$dst))), + (not (i32 (atomic_load_nonext_32 addr:$dst))), + (not (i64 (atomic_load_nonext_64 addr:$dst)))>; def : Pat<(atomic_store_8 (i8 imm:$src), addr:$dst), (MOV8mi addr:$dst, imm:$src)>; @@ -1195,10 +1195,10 @@ def : Pat<(atomic_store_32 GR32:$src, addr:$dst), def : Pat<(atomic_store_64 GR64:$src, addr:$dst), (MOV64mr addr:$dst, GR64:$src)>; -def : Pat<(i8 (atomic_load_8 addr:$src)), (MOV8rm addr:$src)>; -def : Pat<(i16 (atomic_load_16 addr:$src)), (MOV16rm addr:$src)>; -def : Pat<(i32 (atomic_load_32 addr:$src)), (MOV32rm addr:$src)>; -def : Pat<(i64 (atomic_load_64 addr:$src)), (MOV64rm addr:$src)>; +def : Pat<(i8 (atomic_load_nonext_8 addr:$src)), (MOV8rm addr:$src)>; +def : Pat<(i16 (atomic_load_nonext_16 addr:$src)), (MOV16rm addr:$src)>; +def : Pat<(i32 (atomic_load_nonext_32 addr:$src)), (MOV32rm addr:$src)>; +def : Pat<(i64 (atomic_load_nonext_64 addr:$src)), (MOV64rm addr:$src)>; // Floating point loads/stores. def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst), @@ -1215,18 +1215,18 @@ def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst), def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst), (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[HasAVX512]>; -def : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), +def : Pat<(f32 (bitconvert (i32 (atomic_load_nonext_32 addr:$src)))), (MOVSSrm_alt addr:$src)>, Requires<[UseSSE1]>; -def : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), +def : Pat<(f32 (bitconvert (i32 (atomic_load_nonext_32 addr:$src)))), (VMOVSSrm_alt addr:$src)>, Requires<[UseAVX]>; -def : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), +def : Pat<(f32 (bitconvert (i32 (atomic_load_nonext_32 addr:$src)))), (VMOVSSZrm_alt addr:$src)>, Requires<[HasAVX512]>; -def : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))), +def : Pat<(f64 (bitconvert (i64 (atomic_load_nonext_64 addr:$src)))), (MOVSDrm_alt addr:$src)>, Requires<[UseSSE2]>; -def : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))), +def : Pat<(f64 (bitconvert (i64 (atomic_load_nonext_64 addr:$src)))), (VMOVSDrm_alt addr:$src)>, Requires<[UseAVX]>; -def : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))), +def : Pat<(f64 (bitconvert (i64 (atomic_load_nonext_64 addr:$src)))), (VMOVSDZrm_alt addr:$src)>, Requires<[HasAVX512]>; //===----------------------------------------------------------------------===//