[LegalizeTypes] Improve ExpandIntRes_XMULO codegen.

topperc · topperc · commit e745f7c56373 · 2021-03-01T09:54:32.000-08:00
The code previously used two BUILD_PAIRs to concatenate the two UMULO results with 0s in the lower bits to match original VT. Then it created an ADD and a UADDO with the original bit width. Each of those operations need to be expanded since they have illegal types. Since we put 0s in the lower bits before the ADD, the lower half of the ADD result will be 0. So the lower half of the UADDO result is solely determined by the other operand. Since the UADDO need to be split in half, we don't really needd an operation for the lower bits. Unfortunately, we don't see that in type legalization and end up creating something more complicated and DAG combine or lowering aren't always able to recover it. This patch directly generates the narrower ADD and UADDO to avoid needing to legalize them. Now only the MUL is done on the original type. Reviewed By: RKSimon Differential Revision: https://reviews.llvm.org/D97440
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -3939,33 +3939,32 @@ void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
     // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
     // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
     // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
-    // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
-    // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
+    // %4 = add iNh %1.0, %2.0 as iN
+    // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
     //
-    // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
+    // %lo = %3.LO
+    // %hi = %5.0
+    // %ovf = %0 || %1.1 || %2.1 || %5.1
     SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
     SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
     GetExpandedInteger(LHS, LHSLow, LHSHigh);
     GetExpandedInteger(RHS, RHSLow, RHSHigh);
     EVT HalfVT = LHSLow.getValueType();
     EVT BitVT = N->getValueType(1);
-    SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
-    SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
+    SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);
 
     SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
     SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
       DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
       DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
 
-    SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
+    SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow);
     Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
-    SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
-                                    One.getValue(0));
 
-    SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
+    SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow);
     Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
-    SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
-                                    Two.getValue(0));
+
+    SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two);
 
     // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
     // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
@@ -3976,10 +3975,10 @@ void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
     SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
       DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
       DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
-    SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
-    SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
-    Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
-    SplitInteger(Five, Lo, Hi);
+    SplitInteger(Three, Lo, Hi);
+
+    Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum);
+    Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1));
     ReplaceValueWith(SDValue(N, 1), Overflow);
     return;
   }
diff --git a/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll
@@ -4,31 +4,27 @@
 define { i128, i8 } @muloti_test(i128 %l, i128 %r) unnamed_addr #0 {
 ; AARCH-LABEL: muloti_test:
 ; AARCH:       // %bb.0: // %start
-; AARCH-NEXT:    mul x8, x3, x0
-; AARCH-NEXT:    umulh x9, x0, x2
-; AARCH-NEXT:    madd x11, x1, x2, x8
-; AARCH-NEXT:    add x8, x9, x11
-; AARCH-NEXT:    cmp x8, x9
-; AARCH-NEXT:    cset w9, lo
-; AARCH-NEXT:    cmp x11, #0 // =0
-; AARCH-NEXT:    csel w9, wzr, w9, eq
 ; AARCH-NEXT:    cmp x3, #0 // =0
-; AARCH-NEXT:    umulh x10, x1, x2
-; AARCH-NEXT:    cset w12, ne
+; AARCH-NEXT:    umulh x8, x1, x2
+; AARCH-NEXT:    cset w10, ne
 ; AARCH-NEXT:    cmp x1, #0 // =0
-; AARCH-NEXT:    umulh x11, x3, x0
-; AARCH-NEXT:    cset w13, ne
-; AARCH-NEXT:    cmp xzr, x10
-; AARCH-NEXT:    and w10, w13, w12
-; AARCH-NEXT:    cset w12, ne
-; AARCH-NEXT:    cmp xzr, x11
-; AARCH-NEXT:    orr w10, w10, w12
+; AARCH-NEXT:    mul x9, x3, x0
 ; AARCH-NEXT:    cset w11, ne
+; AARCH-NEXT:    cmp xzr, x8
+; AARCH-NEXT:    umulh x8, x3, x0
+; AARCH-NEXT:    madd x9, x1, x2, x9
+; AARCH-NEXT:    and w10, w11, w10
+; AARCH-NEXT:    cset w11, ne
+; AARCH-NEXT:    cmp xzr, x8
+; AARCH-NEXT:    umulh x8, x0, x2
 ; AARCH-NEXT:    orr w10, w10, w11
-; AARCH-NEXT:    orr w9, w10, w9
+; AARCH-NEXT:    cset w11, ne
+; AARCH-NEXT:    adds x1, x8, x9
+; AARCH-NEXT:    orr w8, w10, w11
+; AARCH-NEXT:    cset w9, hs
+; AARCH-NEXT:    orr w8, w8, w9
 ; AARCH-NEXT:    mul x0, x0, x2
-; AARCH-DAG:    mov x1, x8
-; AARCH-DAG:    mov w2, w9
+; AARCH-NEXT:    mov w2, w8
 ; AARCH-NEXT:    ret
 start:
   %0 = tail call { i128, i1 } @llvm.umul.with.overflow.i128(i128 %l, i128 %r) #2
diff --git a/llvm/test/CodeGen/AArch64/vec_umulo.ll b/llvm/test/CodeGen/AArch64/vec_umulo.ll
@@ -316,59 +316,53 @@ define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
 ; CHECK-LABEL: umulo_v2i128:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mul x9, x7, x2
-; CHECK-NEXT:    umulh x10, x2, x6
-; CHECK-NEXT:    madd x9, x3, x6, x9
-; CHECK-NEXT:    add x15, x10, x9
-; CHECK-NEXT:    cmp x15, x10
-; CHECK-NEXT:    cset w10, lo
-; CHECK-NEXT:    cmp x9, #0 // =0
-; CHECK-NEXT:    csel w10, wzr, w10, eq
 ; CHECK-NEXT:    cmp x7, #0 // =0
-; CHECK-NEXT:    umulh x11, x3, x6
-; CHECK-NEXT:    mul x13, x5, x0
-; CHECK-NEXT:    cset w17, ne
+; CHECK-NEXT:    umulh x8, x3, x6
+; CHECK-NEXT:    cset w13, ne
 ; CHECK-NEXT:    cmp x3, #0 // =0
-; CHECK-NEXT:    umulh x12, x7, x2
-; CHECK-NEXT:    umulh x9, x0, x4
-; CHECK-NEXT:    madd x13, x1, x4, x13
-; CHECK-NEXT:    cset w18, ne
-; CHECK-NEXT:    cmp xzr, x11
+; CHECK-NEXT:    umulh x9, x7, x2
+; CHECK-NEXT:    mul x10, x7, x2
+; CHECK-NEXT:    cset w14, ne
+; CHECK-NEXT:    cmp xzr, x8
 ; CHECK-NEXT:    ldr x8, [sp]
-; CHECK-NEXT:    add x11, x9, x13
-; CHECK-NEXT:    and w17, w18, w17
-; CHECK-NEXT:    cset w18, ne
-; CHECK-NEXT:    cmp xzr, x12
-; CHECK-NEXT:    orr w12, w17, w18
-; CHECK-NEXT:    cset w17, ne
-; CHECK-NEXT:    cmp x11, x9
-; CHECK-NEXT:    orr w9, w12, w17
-; CHECK-NEXT:    cset w12, lo
-; CHECK-NEXT:    cmp x13, #0 // =0
-; CHECK-NEXT:    mul x14, x2, x6
-; CHECK-NEXT:    csel w12, wzr, w12, eq
-; CHECK-NEXT:    cmp x5, #0 // =0
-; CHECK-NEXT:    stp x14, x15, [x8, #16]
-; CHECK-NEXT:    umulh x14, x1, x4
-; CHECK-NEXT:    cset w13, ne
-; CHECK-NEXT:    cmp x1, #0 // =0
-; CHECK-NEXT:    umulh x16, x5, x0
-; CHECK-NEXT:    cset w17, ne
-; CHECK-NEXT:    cmp xzr, x14
-; CHECK-NEXT:    and w13, w17, w13
+; CHECK-NEXT:    umulh x11, x2, x6
+; CHECK-NEXT:    madd x10, x3, x6, x10
+; CHECK-NEXT:    and w13, w14, w13
 ; CHECK-NEXT:    cset w14, ne
-; CHECK-NEXT:    cmp xzr, x16
+; CHECK-NEXT:    cmp xzr, x9
 ; CHECK-NEXT:    orr w13, w13, w14
 ; CHECK-NEXT:    cset w14, ne
+; CHECK-NEXT:    adds x10, x11, x10
+; CHECK-NEXT:    mul x12, x2, x6
 ; CHECK-NEXT:    orr w13, w13, w14
-; CHECK-NEXT:    orr w12, w13, w12
+; CHECK-NEXT:    cset w14, hs
+; CHECK-NEXT:    cmp x5, #0 // =0
+; CHECK-NEXT:    umulh x17, x1, x4
+; CHECK-NEXT:    stp x12, x10, [x8, #16]
+; CHECK-NEXT:    cset w10, ne
+; CHECK-NEXT:    cmp x1, #0 // =0
+; CHECK-NEXT:    umulh x9, x5, x0
+; CHECK-NEXT:    mul x11, x5, x0
+; CHECK-NEXT:    cset w12, ne
+; CHECK-NEXT:    cmp xzr, x17
+; CHECK-NEXT:    umulh x15, x0, x4
+; CHECK-NEXT:    madd x11, x1, x4, x11
+; CHECK-NEXT:    and w10, w12, w10
+; CHECK-NEXT:    cset w12, ne
+; CHECK-NEXT:    cmp xzr, x9
+; CHECK-NEXT:    orr w9, w10, w12
+; CHECK-NEXT:    cset w10, ne
+; CHECK-NEXT:    adds x11, x15, x11
+; CHECK-NEXT:    orr w9, w9, w10
+; CHECK-NEXT:    cset w10, hs
 ; CHECK-NEXT:    orr w9, w9, w10
-; CHECK-NEXT:    fmov s0, w12
-; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    orr w10, w13, w14
+; CHECK-NEXT:    fmov s0, w9
+; CHECK-NEXT:    mov v0.s[1], w10
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #31
-; CHECK-NEXT:    mul x15, x0, x4
+; CHECK-NEXT:    mul x16, x0, x4
 ; CHECK-NEXT:    sshr v0.2s, v0.2s, #31
-; CHECK-NEXT:    stp x15, x11, [x8]
+; CHECK-NEXT:    stp x16, x11, [x8]
 ; CHECK-NEXT:    ret
   %t = call {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
   %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/PowerPC/pr45448.ll b/llvm/test/CodeGen/PowerPC/pr45448.ll
@@ -23,17 +23,14 @@ define hidden void @julia_tryparse_internal_45896() #0 {
 ; CHECK-NEXT:    cmpdi r3, 0
 ; CHECK-NEXT:    sradi r4, r3, 63
 ; CHECK-NEXT:    rldic r5, r5, 4, 32
-; CHECK-NEXT:    crnot 4*cr5+gt, eq
+; CHECK-NEXT:    crnot 4*cr5+lt, eq
 ; CHECK-NEXT:    mulhdu r3, r3, r5
 ; CHECK-NEXT:    maddld r6, r4, r5, r3
-; CHECK-NEXT:    cmpld r6, r3
-; CHECK-NEXT:    mulld r3, r4, r5
-; CHECK-NEXT:    cmpldi cr1, r3, 0
-; CHECK-NEXT:    crandc 4*cr5+lt, lt, 4*cr1+eq
+; CHECK-NEXT:    cmpld cr1, r6, r3
 ; CHECK-NEXT:    mulhdu. r3, r4, r5
-; CHECK-NEXT:    bc 4, 4*cr5+gt, .LBB0_10
+; CHECK-NEXT:    bc 4, 4*cr5+lt, .LBB0_10
 ; CHECK-NEXT:  # %bb.8: # %L670
-; CHECK-NEXT:    crorc 4*cr5+lt, 4*cr5+lt, eq
+; CHECK-NEXT:    crorc 4*cr5+lt, 4*cr1+lt, eq
 ; CHECK-NEXT:    bc 4, 4*cr5+lt, .LBB0_10
 ; CHECK-NEXT:  # %bb.9: # %L917
 ; CHECK-NEXT:  .LBB0_10: # %L994
diff --git a/llvm/test/CodeGen/PowerPC/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/PowerPC/umulo-128-legalisation-lowering.ll
diff --git a/llvm/test/CodeGen/SPARC/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/SPARC/umulo-128-legalisation-lowering.ll
