[X86] Disable lowering constant build vectors to broadcasts on AVX512 targets

On AVX512 targets we're better off keeping constant vector at full width to ensure that they can be load folded into vector instructions, reducing register pressure.

If a vector constant remains as a basic load, X86FixupVectorConstantsPass will still convert this to a broadcast instruction for us.

Non-VLX targets are still seeing some regressions due to these being implicitly widened to 512-bit ops in isel patterns and not in the DAG, so I've limited this to just 512-bit vectors for now.

Author: RKSimon

llvm/lib/Target/X86/X86FixupVectorConstants.cpp

@@ -407,6 +407,26 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF,
                           {X86::VMOVSDZrm, 1, 64, rebuildZeroUpperCst},
                           {X86::VMOVDDUPZ128rm, 1, 64, rebuildSplatCst}},
                          128, 1);
+  case X86::VMOVAPDZ128rmk:
+  case X86::VMOVUPDZ128rmk:
+    return FixupConstant({{X86::VMOVSDZrmk, 1, 64, rebuildZeroUpperCst},
+                          {X86::VMOVDDUPZ128rmk, 1, 64, rebuildSplatCst}},
+                         128, 3);
+  case X86::VMOVAPDZ128rmkz:
+  case X86::VMOVUPDZ128rmkz:
+    return FixupConstant({{X86::VMOVSDZrmkz, 1, 64, rebuildZeroUpperCst},
+                          {X86::VMOVDDUPZ128rmkz, 1, 64, rebuildSplatCst}},
+                         128, 2);
+  case X86::VMOVAPSZ128rmk:
+  case X86::VMOVUPSZ128rmk:
+    return FixupConstant({{X86::VMOVSSZrmk, 1, 32, rebuildZeroUpperCst},
+                          {X86::VBROADCASTSSZ128rmk, 1, 32, rebuildSplatCst}},
+                         128, 3);
+  case X86::VMOVAPSZ128rmkz:
+  case X86::VMOVUPSZ128rmkz:
+    return FixupConstant({{X86::VMOVSSZrmkz, 1, 32, rebuildZeroUpperCst},
+                          {X86::VBROADCASTSSZ128rmkz, 1, 32, rebuildSplatCst}},
+                         128, 2);
   case X86::VMOVAPDZ256rm:
   case X86::VMOVAPSZ256rm:
   case X86::VMOVUPDZ256rm:
@@ -416,6 +436,26 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF,
          {X86::VBROADCASTSDZ256rm, 1, 64, rebuildSplatCst},
          {X86::VBROADCASTF32X4Z256rm, 1, 128, rebuildSplatCst}},
         256, 1);
+  case X86::VMOVAPDZ256rmk:
+  case X86::VMOVUPDZ256rmk:
+    return FixupConstant({{X86::VBROADCASTSDZ256rmk, 1, 64, rebuildSplatCst}},
+                         256, 3);
+  case X86::VMOVAPDZ256rmkz:
+  case X86::VMOVUPDZ256rmkz:
+    return FixupConstant({{X86::VBROADCASTSDZ256rmkz, 1, 64, rebuildSplatCst}},
+                         256, 2);
+  case X86::VMOVAPSZ256rmk:
+  case X86::VMOVUPSZ256rmk:
+    return FixupConstant(
+        {{X86::VBROADCASTSSZ256rmk, 1, 32, rebuildSplatCst},
+         {X86::VBROADCASTF32X4Z256rmk, 1, 128, rebuildSplatCst}},
+        256, 3);
+  case X86::VMOVAPSZ256rmkz:
+  case X86::VMOVUPSZ256rmkz:
+    return FixupConstant(
+        {{X86::VBROADCASTSSZ256rmkz, 1, 32, rebuildSplatCst},
+         {X86::VBROADCASTF32X4Z256rmkz, 1, 128, rebuildSplatCst}},
+        256, 2);
   case X86::VMOVAPDZrm:
   case X86::VMOVAPSZrm:
   case X86::VMOVUPDZrm:
@@ -425,6 +465,26 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF,
                           {X86::VBROADCASTF32X4rm, 1, 128, rebuildSplatCst},
                           {X86::VBROADCASTF64X4rm, 1, 256, rebuildSplatCst}},
                          512, 1);
+  case X86::VMOVAPDZrmk:
+  case X86::VMOVUPDZrmk:
+    return FixupConstant({{X86::VBROADCASTSDZrmk, 1, 64, rebuildSplatCst},
+                          {X86::VBROADCASTF64X4rmk, 1, 256, rebuildSplatCst}},
+                         512, 3);
+  case X86::VMOVAPDZrmkz:
+  case X86::VMOVUPDZrmkz:
+    return FixupConstant({{X86::VBROADCASTSDZrmkz, 1, 64, rebuildSplatCst},
+                          {X86::VBROADCASTF64X4rmkz, 1, 256, rebuildSplatCst}},
+                         512, 2);
+  case X86::VMOVAPSZrmk:
+  case X86::VMOVUPSZrmk:
+    return FixupConstant({{X86::VBROADCASTSSZrmk, 1, 32, rebuildSplatCst},
+                          {X86::VBROADCASTF32X4rmk, 1, 128, rebuildSplatCst}},
+                         512, 3);
+  case X86::VMOVAPSZrmkz:
+  case X86::VMOVUPSZrmkz:
+    return FixupConstant({{X86::VBROADCASTSSZrmkz, 1, 32, rebuildSplatCst},
+                          {X86::VBROADCASTF32X4rmkz, 1, 128, rebuildSplatCst}},
+                         512, 2);
     /* Integer Loads */
   case X86::MOVDQArm:
   case X86::MOVDQUrm: {
@@ -520,6 +580,42 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF,
         {X86::VPMOVZXDQZ128rm, 2, 32, rebuildZExtCst}};
     return FixupConstant(Fixups, 128, 1);
   }
+  case X86::VMOVDQA32Z128rmk:
+  case X86::VMOVDQU32Z128rmk:
+    return FixupConstant({{X86::VPBROADCASTDZ128rmk, 1, 32, rebuildSplatCst},
+                          {X86::VPMOVSXBDZ128rmk, 4, 8, rebuildSExtCst},
+                          {X86::VPMOVZXBDZ128rmk, 4, 8, rebuildZExtCst},
+                          {X86::VPMOVSXWDZ128rmk, 4, 16, rebuildSExtCst},
+                          {X86::VPMOVZXWDZ128rmk, 4, 16, rebuildZExtCst}},
+                         128, 3);
+  case X86::VMOVDQA32Z128rmkz:
+  case X86::VMOVDQU32Z128rmkz:
+    return FixupConstant({{X86::VPBROADCASTDZ128rmkz, 1, 32, rebuildSplatCst},
+                          {X86::VPMOVSXBDZ128rmkz, 4, 8, rebuildSExtCst},
+                          {X86::VPMOVZXBDZ128rmkz, 4, 8, rebuildZExtCst},
+                          {X86::VPMOVSXWDZ128rmkz, 4, 16, rebuildSExtCst},
+                          {X86::VPMOVZXWDZ128rmkz, 4, 16, rebuildZExtCst}},
+                         128, 2);
+  case X86::VMOVDQA64Z128rmk:
+  case X86::VMOVDQU64Z128rmk:
+    return FixupConstant({{X86::VPMOVSXBQZ128rmk, 2, 8, rebuildSExtCst},
+                          {X86::VPMOVZXBQZ128rmk, 2, 8, rebuildZExtCst},
+                          {X86::VPMOVSXWQZ128rmk, 2, 16, rebuildSExtCst},
+                          {X86::VPMOVZXWQZ128rmk, 2, 16, rebuildZExtCst},
+                          {X86::VPBROADCASTQZ128rmk, 1, 64, rebuildSplatCst},
+                          {X86::VPMOVSXDQZ128rmk, 2, 32, rebuildSExtCst},
+                          {X86::VPMOVZXDQZ128rmk, 2, 32, rebuildZExtCst}},
+                         128, 3);
+  case X86::VMOVDQA64Z128rmkz:
+  case X86::VMOVDQU64Z128rmkz:
+    return FixupConstant({{X86::VPMOVSXBQZ128rmkz, 2, 8, rebuildSExtCst},
+                          {X86::VPMOVZXBQZ128rmkz, 2, 8, rebuildZExtCst},
+                          {X86::VPMOVSXWQZ128rmkz, 2, 16, rebuildSExtCst},
+                          {X86::VPMOVZXWQZ128rmkz, 2, 16, rebuildZExtCst},
+                          {X86::VPBROADCASTQZ128rmkz, 1, 64, rebuildSplatCst},
+                          {X86::VPMOVSXDQZ128rmkz, 2, 32, rebuildSExtCst},
+                          {X86::VPMOVZXDQZ128rmkz, 2, 32, rebuildZExtCst}},
+                         128, 2);
   case X86::VMOVDQA32Z256rm:
   case X86::VMOVDQA64Z256rm:
   case X86::VMOVDQU32Z256rm:
@@ -544,6 +640,46 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF,
         {X86::VPMOVZXDQZ256rm, 4, 32, rebuildZExtCst}};
     return FixupConstant(Fixups, 256, 1);
   }
+  case X86::VMOVDQA32Z256rmk:
+  case X86::VMOVDQU32Z256rmk:
+    return FixupConstant(
+        {{X86::VPBROADCASTDZ256rmk, 1, 32, rebuildSplatCst},
+         {X86::VPMOVSXBDZ256rmk, 8, 8, rebuildSExtCst},
+         {X86::VPMOVZXBDZ256rmk, 8, 8, rebuildZExtCst},
+         {X86::VBROADCASTI32X4Z256rmk, 1, 128, rebuildSplatCst},
+         {X86::VPMOVSXWDZ256rmk, 8, 16, rebuildSExtCst},
+         {X86::VPMOVZXWDZ256rmk, 8, 16, rebuildZExtCst}},
+        256, 3);
+  case X86::VMOVDQA32Z256rmkz:
+  case X86::VMOVDQU32Z256rmkz:
+    return FixupConstant(
+        {{X86::VPBROADCASTDZ256rmkz, 1, 32, rebuildSplatCst},
+         {X86::VPMOVSXBDZ256rmkz, 8, 8, rebuildSExtCst},
+         {X86::VPMOVZXBDZ256rmkz, 8, 8, rebuildZExtCst},
+         {X86::VBROADCASTI32X4Z256rmkz, 1, 128, rebuildSplatCst},
+         {X86::VPMOVSXWDZ256rmkz, 8, 16, rebuildSExtCst},
+         {X86::VPMOVZXWDZ256rmkz, 8, 16, rebuildZExtCst}},
+        256, 2);
+  case X86::VMOVDQA64Z256rmk:
+  case X86::VMOVDQU64Z256rmk:
+    return FixupConstant({{X86::VPMOVSXBQZ256rmk, 4, 8, rebuildSExtCst},
+                          {X86::VPMOVZXBQZ256rmk, 4, 8, rebuildZExtCst},
+                          {X86::VPBROADCASTQZ256rmk, 1, 64, rebuildSplatCst},
+                          {X86::VPMOVSXWQZ256rmk, 4, 16, rebuildSExtCst},
+                          {X86::VPMOVZXWQZ256rmk, 4, 16, rebuildZExtCst},
+                          {X86::VPMOVSXDQZ256rmk, 4, 32, rebuildSExtCst},
+                          {X86::VPMOVZXDQZ256rmk, 4, 32, rebuildZExtCst}},
+                         256, 3);
+  case X86::VMOVDQA64Z256rmkz:
+  case X86::VMOVDQU64Z256rmkz:
+    return FixupConstant({{X86::VPMOVSXBQZ256rmkz, 4, 8, rebuildSExtCst},
+                          {X86::VPMOVZXBQZ256rmkz, 4, 8, rebuildZExtCst},
+                          {X86::VPBROADCASTQZ256rmkz, 1, 64, rebuildSplatCst},
+                          {X86::VPMOVSXWQZ256rmkz, 4, 16, rebuildSExtCst},
+                          {X86::VPMOVZXWQZ256rmkz, 4, 16, rebuildZExtCst},
+                          {X86::VPMOVSXDQZ256rmkz, 4, 32, rebuildSExtCst},
+                          {X86::VPMOVZXDQZ256rmkz, 4, 32, rebuildZExtCst}},
+                         256, 2);
   case X86::VMOVDQA32Zrm:
   case X86::VMOVDQA64Zrm:
   case X86::VMOVDQU32Zrm:
@@ -569,43 +705,93 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF,
         {X86::VPMOVZXDQZrm, 8, 32, rebuildZExtCst}};
     return FixupConstant(Fixups, 512, 1);
   }
+  case X86::VMOVDQA32Zrmk:
+  case X86::VMOVDQU32Zrmk:
+    return FixupConstant({{X86::VPBROADCASTDZrmk, 1, 32, rebuildSplatCst},
+                          {X86::VBROADCASTI32X4rmk, 1, 128, rebuildSplatCst},
+                          {X86::VPMOVSXBDZrmk, 16, 8, rebuildSExtCst},
+                          {X86::VPMOVZXBDZrmk, 16, 8, rebuildZExtCst},
+                          {X86::VPMOVSXWDZrmk, 16, 16, rebuildSExtCst},
+                          {X86::VPMOVZXWDZrmk, 16, 16, rebuildZExtCst}},
+                         512, 3);
+  case X86::VMOVDQA32Zrmkz:
+  case X86::VMOVDQU32Zrmkz:
+    return FixupConstant({{X86::VPBROADCASTDZrmkz, 1, 32, rebuildSplatCst},
+                          {X86::VBROADCASTI32X4rmkz, 1, 128, rebuildSplatCst},
+                          {X86::VPMOVSXBDZrmkz, 16, 8, rebuildSExtCst},
+                          {X86::VPMOVZXBDZrmkz, 16, 8, rebuildZExtCst},
+                          {X86::VPMOVSXWDZrmkz, 16, 16, rebuildSExtCst},
+                          {X86::VPMOVZXWDZrmkz, 16, 16, rebuildZExtCst}},
+                         512, 2);
+  case X86::VMOVDQA64Zrmk:
+  case X86::VMOVDQU64Zrmk:
+    return FixupConstant({{X86::VPBROADCASTQZrmk, 1, 64, rebuildSplatCst},
+                          {X86::VPMOVSXBQZrmk, 8, 8, rebuildSExtCst},
+                          {X86::VPMOVZXBQZrmk, 8, 8, rebuildZExtCst},
+                          {X86::VPMOVSXWQZrmk, 8, 16, rebuildSExtCst},
+                          {X86::VPMOVZXWQZrmk, 8, 16, rebuildZExtCst},
+                          {X86::VBROADCASTI64X4rmk, 1, 256, rebuildSplatCst},
+                          {X86::VPMOVSXDQZrmk, 8, 32, rebuildSExtCst},
+                          {X86::VPMOVZXDQZrmk, 8, 32, rebuildZExtCst}},
+                         512, 3);
+  case X86::VMOVDQA64Zrmkz:
+  case X86::VMOVDQU64Zrmkz:
+    return FixupConstant({{X86::VPBROADCASTQZrmkz, 1, 64, rebuildSplatCst},
+                          {X86::VPMOVSXBQZrmkz, 8, 8, rebuildSExtCst},
+                          {X86::VPMOVZXBQZrmkz, 8, 8, rebuildZExtCst},
+                          {X86::VPMOVSXWQZrmkz, 8, 16, rebuildSExtCst},
+                          {X86::VPMOVZXWQZrmkz, 8, 16, rebuildZExtCst},
+                          {X86::VBROADCASTI64X4rmkz, 1, 256, rebuildSplatCst},
+                          {X86::VPMOVSXDQZrmkz, 8, 32, rebuildSExtCst},
+                          {X86::VPMOVZXDQZrmkz, 8, 32, rebuildZExtCst}},
+                         512, 2);
   }
 
-  auto ConvertToBroadcastAVX512 = [&](unsigned OpSrc32, unsigned OpSrc64) {
-    unsigned OpBcst32 = 0, OpBcst64 = 0;
-    unsigned OpNoBcst32 = 0, OpNoBcst64 = 0;
+  auto ConvertToBroadcastAVX512 = [&](unsigned OpSrc16, unsigned OpSrc32,
+                                      unsigned OpSrc64) {
+    if (OpSrc16) {
+      if (const X86FoldTableEntry *Mem2Bcst =
+              llvm::lookupBroadcastFoldTableBySize(OpSrc16, 16)) {
+        unsigned OpBcst16 = Mem2Bcst->DstOp;
+        unsigned OpNoBcst16 = Mem2Bcst->Flags & TB_INDEX_MASK;
+        FixupEntry Fixups[] = {{(int)OpBcst16, 1, 16, rebuildSplatCst}};
+        // TODO: Add support for RegBitWidth, but currently rebuildSplatCst
+        // doesn't require it (defaults to Constant::getPrimitiveSizeInBits).
+        if (FixupConstant(Fixups, 0, OpNoBcst16))
+          return true;
+      }
+    }
     if (OpSrc32) {
       if (const X86FoldTableEntry *Mem2Bcst =
               llvm::lookupBroadcastFoldTableBySize(OpSrc32, 32)) {
-        OpBcst32 = Mem2Bcst->DstOp;
-        OpNoBcst32 = Mem2Bcst->Flags & TB_INDEX_MASK;
+        unsigned OpBcst32 = Mem2Bcst->DstOp;
+        unsigned OpNoBcst32 = Mem2Bcst->Flags & TB_INDEX_MASK;
+        FixupEntry Fixups[] = {{(int)OpBcst32, 1, 32, rebuildSplatCst}};
+        // TODO: Add support for RegBitWidth, but currently rebuildSplatCst
+        // doesn't require it (defaults to Constant::getPrimitiveSizeInBits).
+        if (FixupConstant(Fixups, 0, OpNoBcst32))
+          return true;
       }
     }
     if (OpSrc64) {
       if (const X86FoldTableEntry *Mem2Bcst =
               llvm::lookupBroadcastFoldTableBySize(OpSrc64, 64)) {
-        OpBcst64 = Mem2Bcst->DstOp;
-        OpNoBcst64 = Mem2Bcst->Flags & TB_INDEX_MASK;
+        unsigned OpBcst64 = Mem2Bcst->DstOp;
+        unsigned OpNoBcst64 = Mem2Bcst->Flags & TB_INDEX_MASK;
+        FixupEntry Fixups[] = {{(int)OpBcst64, 1, 64, rebuildSplatCst}};
+        // TODO: Add support for RegBitWidth, but currently rebuildSplatCst
+        // doesn't require it (defaults to Constant::getPrimitiveSizeInBits).
+        if (FixupConstant(Fixups, 0, OpNoBcst64))
+          return true;
       }
     }
-    assert(((OpBcst32 == 0) || (OpBcst64 == 0) || (OpNoBcst32 == OpNoBcst64)) &&
-           "OperandNo mismatch");
-
-    if (OpBcst32 || OpBcst64) {
-      unsigned OpNo = OpBcst32 == 0 ? OpNoBcst64 : OpNoBcst32;
-      FixupEntry Fixups[] = {{(int)OpBcst32, 32, 32, rebuildSplatCst},
-                             {(int)OpBcst64, 64, 64, rebuildSplatCst}};
-      // TODO: Add support for RegBitWidth, but currently rebuildSplatCst
-      // doesn't require it (defaults to Constant::getPrimitiveSizeInBits).
-      return FixupConstant(Fixups, 0, OpNo);
-    }
     return false;
   };
 
   // Attempt to find a AVX512 mapping from a full width memory-fold instruction
   // to a broadcast-fold instruction variant.
   if ((MI.getDesc().TSFlags & X86II::EncodingMask) == X86II::EVEX)
-    return ConvertToBroadcastAVX512(Opc, Opc);
+    return ConvertToBroadcastAVX512(Opc, Opc, Opc);
 
   // Reverse the X86InstrInfo::setExecutionDomainCustom EVEX->VEX logic
   // conversion to see if we can convert to a broadcasted (integer) logic op.
@@ -662,7 +848,7 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF,
       break;
     }
     if (OpSrc32 || OpSrc64)
-      return ConvertToBroadcastAVX512(OpSrc32, OpSrc64);
+      return ConvertToBroadcastAVX512(0, OpSrc32, OpSrc64);
   }
 
   return false;

llvm/lib/Target/X86/X86ISelLowering.cpp

@@ -7227,6 +7227,14 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
          "Unsupported vector type for broadcast.");
 
+  // On AVX512VL targets we're better off keeping the full width constant load
+  // and letting X86FixupVectorConstantsPass handle conversion to
+  // broadcast/broadcast-fold.
+  // AVX512 targets without AVX512VL can do this only for 512-bit vectors.
+  if (Subtarget.hasAVX512() && (Subtarget.hasVLX() || VT.is512BitVector()) &&
+      BVOp->isConstant())
+    return SDValue();
+
   // See if the build vector is a repeating sequence of scalars (inc. splat).
   SDValue Ld;
   BitVector UndefElements;

llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll

@@ -1480,10 +1480,8 @@ define void @vec256_i8_widen_to_i128_factor16_broadcast_to_v2i128_factor2(ptr %i
 ; AVX512F-NEXT:    vpaddb 32(%rsi), %ymm1, %ymm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpternlogq $202, %ymm0, %ymm1, %ymm2
-; AVX512F-NEXT:    vpaddb (%rdx), %ymm2, %ymm0
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1495,10 +1493,8 @@ define void @vec256_i8_widen_to_i128_factor16_broadcast_to_v2i128_factor2(ptr %i
 ; AVX512DQ-NEXT:    vpaddb 32(%rsi), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm0, %ymm1, %ymm2
-; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
@@ -3260,10 +3256,8 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,1,0,1]
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpternlogq $202, %ymm2, %ymm1, %ymm3
-; AVX512F-NEXT:    vpaddb (%rdx), %ymm3, %ymm1
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX512F-NEXT:    vpaddb (%rdx), %ymm2, %ymm1
 ; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)
 ; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)
@@ -3277,10 +3271,8 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,1,0,1]
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm2, %ymm1, %ymm3
-; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm3, %ymm1
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm2, %ymm1
 ; AVX512DQ-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, 32(%rcx)
 ; AVX512DQ-NEXT:    vmovdqa %ymm1, (%rcx)