[InstSimplify] recognize trunc + icmp sgt/slt variants of select simplifications (PR28466)

rL245171 exposed a hole in InstSimplify that manifested in a strange way in PR28466:
https://llvm.org/bugs/show_bug.cgi?id=28466

It's possible to use trunc + icmp sgt/slt in place of an and + icmp eq/ne, so we need to
recognize that pattern to eliminate selects that are choosing between some value and some
bitmasked version of that value.

Note that there is significant room for improvement (refactoring) and enhancement (more
patterns, possibly in InstCombine rather than here).

Differential Revision: https://reviews.llvm.org/D22537



git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@276341 91177308-0d34-0410-b5e6-96231b3b80d8

Author: rotateright

lib/Analysis/InstructionSimplify.cpp

@@ -3406,6 +3406,33 @@ static Value *simplifySelectBitTest(Value *TrueVal, Value *FalseVal, Value *X,
   return nullptr;
 }
 
+/// An alternative way to test if a bit is set or not uses sgt/slt instead of
+/// eq/ne.
+static Value *simplifySelectWithFakeICmpEq(Value *CmpLHS, Value *TrueVal,
+                                           Value *FalseVal,
+                                           bool TrueWhenUnset) {
+  unsigned BitWidth = TrueVal->getType()->getScalarSizeInBits();
+  APInt MinSignedValue;
+  Value *X;
+  if (match(CmpLHS, m_Trunc(m_Value(X))) && (X == TrueVal || X == FalseVal)) {
+    // icmp slt (trunc X), 0  <--> icmp ne (and X, C), 0
+    // icmp sgt (trunc X), -1 <--> icmp eq (and X, C), 0
+    unsigned DestSize = CmpLHS->getType()->getScalarSizeInBits();
+    MinSignedValue = APInt::getSignedMinValue(DestSize).zext(BitWidth);
+  } else {
+    // icmp slt X, 0  <--> icmp ne (and X, C), 0
+    // icmp sgt X, -1 <--> icmp eq (and X, C), 0
+    X = CmpLHS;
+    MinSignedValue = APInt::getSignedMinValue(BitWidth);
+  }
+
+  if (Value *V = simplifySelectBitTest(TrueVal, FalseVal, X, &MinSignedValue,
+                                       TrueWhenUnset))
+    return V;
+
+  return nullptr;
+}
+
 /// Try to simplify a select instruction when its condition operand is an
 /// integer comparison.
 static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal,
@@ -3418,9 +3445,6 @@ static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal,
 
   // FIXME: This code is nearly duplicated in InstCombine. Using/refactoring
   // decomposeBitTestICmp() might help.
-  unsigned BitWidth =
-      Q.DL.getTypeSizeInBits(TrueVal->getType()->getScalarType());
-  APInt MinSignedValue = APInt::getSignBit(BitWidth);
   if (ICmpInst::isEquality(Pred) && match(CmpRHS, m_Zero())) {
     Value *X;
     const APInt *Y;
@@ -3429,12 +3453,14 @@ static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal,
                                            Pred == ICmpInst::ICMP_EQ))
         return V;
   } else if (Pred == ICmpInst::ICMP_SLT && match(CmpRHS, m_Zero())) {
-    if (Value *V = simplifySelectBitTest(TrueVal, FalseVal, CmpLHS,
-                                         &MinSignedValue, false))
+    // Comparing signed-less-than 0 checks if the sign bit is set.
+    if (Value *V = simplifySelectWithFakeICmpEq(CmpLHS, TrueVal, FalseVal,
+                                                false))
       return V;
   } else if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, m_AllOnes())) {
-    if (Value *V = simplifySelectBitTest(TrueVal, FalseVal, CmpLHS,
-                                         &MinSignedValue, true))
+    // Comparing signed-greater-than -1 checks if the sign bit is not set.
+    if (Value *V = simplifySelectWithFakeICmpEq(CmpLHS, TrueVal, FalseVal,
+                                                true))
       return V;
   }
 

test/Transforms/InstSimplify/select.ll

@@ -228,11 +228,8 @@ define i32 @select_icmp_and_8_ne_0_and_not_8_alt(i32 %x) {
 
 define i32 @select_icmp_trunc_8_ne_0_or_128(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_ne_0_or_128(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 %x to i8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[TRUNC]], -1
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 128
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[OR]], i32 %x
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %trunc = trunc i32 %x to i8
   %cmp = icmp sgt i8 %trunc, -1
@@ -243,11 +240,8 @@ define i32 @select_icmp_trunc_8_ne_0_or_128(i32 %x) {
 
 define i32 @select_icmp_trunc_8_ne_0_or_128_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_ne_0_or_128_alt(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 %x to i8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TRUNC]], 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 128
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 %x, i32 [[OR]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %trunc = trunc i32 %x to i8
   %cmp = icmp slt i8 %trunc, 0
@@ -258,11 +252,7 @@ define i32 @select_icmp_trunc_8_ne_0_or_128_alt(i32 %x) {
 
 define i32 @select_icmp_trunc_8_eq_0_or_128(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_eq_0_or_128(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 %x to i8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TRUNC]], 0
-; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 128
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[OR]], i32 %x
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 %x
 ;
   %trunc = trunc i32 %x to i8
   %cmp = icmp slt i8 %trunc, 0
@@ -273,11 +263,7 @@ define i32 @select_icmp_trunc_8_eq_0_or_128(i32 %x) {
 
 define i32 @select_icmp_trunc_8_eq_0_or_128_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_eq_0_or_128_alt(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 %x to i8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[TRUNC]], -1
-; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 128
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 %x, i32 [[OR]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 %x
 ;
   %trunc = trunc i32 %x to i8
   %cmp = icmp sgt i8 %trunc, -1
@@ -288,11 +274,8 @@ define i32 @select_icmp_trunc_8_eq_0_or_128_alt(i32 %x) {
 
 define i32 @select_icmp_trunc_8_eq_0_and_not_8(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_eq_0_and_not_8(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 %x to i4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i4 [[TRUNC]], -1
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, -9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 %x, i32 [[AND]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %trunc = trunc i32 %x to i4
   %cmp = icmp sgt i4 %trunc, -1
@@ -303,11 +286,8 @@ define i32 @select_icmp_trunc_8_eq_0_and_not_8(i32 %x) {
 
 define i32 @select_icmp_trunc_8_eq_0_and_not_8_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_eq_0_and_not_8_alt(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 %x to i4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i4 [[TRUNC]], 0
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, -9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[AND]], i32 %x
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %trunc = trunc i32 %x to i4
   %cmp = icmp slt i4 %trunc, 0
@@ -318,11 +298,7 @@ define i32 @select_icmp_trunc_8_eq_0_and_not_8_alt(i32 %x) {
 
 define i32 @select_icmp_trunc_8_ne_0_and_not_8(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 %x to i4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i4 [[TRUNC]], 0
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, -9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 %x, i32 [[AND]]
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 %x
 ;
   %trunc = trunc i32 %x to i4
   %cmp = icmp slt i4 %trunc, 0
@@ -333,11 +309,7 @@ define i32 @select_icmp_trunc_8_ne_0_and_not_8(i32 %x) {
 
 define i32 @select_icmp_trunc_8_ne_0_and_not_8_alt(i32 %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8_alt(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 %x to i4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i4 [[TRUNC]], -1
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, -9
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[AND]], i32 %x
-; CHECK-NEXT:    ret i32 [[SEL]]
+; CHECK-NEXT:    ret i32 %x
 ;
   %trunc = trunc i32 %x to i4
   %cmp = icmp sgt i4 %trunc, -1
@@ -361,11 +333,7 @@ define <2 x i32> @select_icmp_and_8_ne_0_and_not_8_vec(<2 x i32> %x) {
 
 define <2 x i32> @select_icmp_trunc_8_ne_0_and_not_8_alt_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8_alt_vec(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i32> %x to <2 x i4>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i4> [[TRUNC]], <i4 -1, i4 -1>
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> %x, <i32 -9, i32 -9>
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[AND]], <2 x i32> %x
-; CHECK-NEXT:    ret <2 x i32> [[SEL]]
+; CHECK-NEXT:    ret <2 x i32> %x
 ;
   %trunc = trunc <2 x i32> %x to <2 x i4>
   %cmp = icmp sgt <2 x i4> %trunc, <i4 -1, i4 -1>