8283091: Support type conversion between different data sizes in SLP

Reviewed-by: kvn, sviswanathan

Author: Fei Gao

src/hotspot/cpu/aarch64/aarch64.ad

@@ -2432,6 +2432,19 @@ const bool Matcher::match_rule_supported(int opcode) {
   return ret_value; // Per default match rules are supported.
 }
 
+const bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+  if (UseSVE == 0) {
+    // ConvD2I and ConvL2F are not profitable to be vectorized on NEON, because no direct
+    // NEON instructions support them. But the match rule support for them is profitable for
+    // Vector API intrinsics.
+    if ((opcode == Op_VectorCastD2X && bt == T_INT) ||
+        (opcode == Op_VectorCastL2X && bt == T_FLOAT)) {
+      return false;
+    }
+  }
+  return match_rule_supported_vector(opcode, vlen, bt);
+}
+
 // Identify extra cases that we might want to provide match rules for vector nodes and
 // other intrinsics guarded with vector length (vlen) and element type (bt).
 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {

src/hotspot/cpu/arm/arm.ad

@@ -981,6 +981,10 @@ const bool Matcher::match_rule_supported(int opcode) {
   return true;  // Per default match rules are supported.
 }
 
+const bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+  return match_rule_supported_vector(opcode, vlen, bt);
+}
+
 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 
   // TODO

src/hotspot/cpu/ppc/ppc.ad

@@ -2165,6 +2165,10 @@ const bool Matcher::match_rule_supported(int opcode) {
   return true; // Per default match rules are supported.
 }
 
+const bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+  return match_rule_supported_vector(opcode, vlen, bt);
+}
+
 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
   if (!match_rule_supported(opcode) || !vector_size_supported(bt, vlen)) {
     return false;

src/hotspot/cpu/riscv/riscv.ad

@@ -1815,6 +1815,10 @@ const bool Matcher::match_rule_supported(int opcode) {
   return true; // Per default match rules are supported.
 }
 
+const bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+  return match_rule_supported_vector(opcode, vlen, bt);
+}
+
 // Identify extra cases that we might want to provide match rules for vector nodes and
 // other intrinsics guarded with vector length (vlen) and element type (bt).
 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {

src/hotspot/cpu/s390/s390.ad

@@ -1505,6 +1505,10 @@ const bool Matcher::match_rule_supported(int opcode) {
   return true; // Per default match rules are supported.
 }
 
+const bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+  return match_rule_supported_vector(opcode, vlen, bt);
+}
+
 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
   if (!match_rule_supported(opcode) || !vector_size_supported(bt, vlen)) {
     return false;

src/hotspot/cpu/x86/x86.ad

@@ -1693,6 +1693,10 @@ static inline bool is_pop_count_instr_target(BasicType bt) {
          (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
 }
 
+const bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+  return match_rule_supported_vector(opcode, vlen, bt);
+}
+
 // Identify extra cases that we might want to provide match rules for vector nodes and
 // other intrinsics guarded with vector length (vlen) and element type (bt).
 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {

src/hotspot/share/opto/matcher.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -325,6 +325,10 @@ class Matcher : public PhaseTransform {
   // should generate this one.
   static const bool match_rule_supported(int opcode);
 
+  // Identify extra cases that we might want to vectorize automatically
+  // And exclude cases which are not profitable to auto-vectorize.
+  static const bool match_rule_supported_superword(int opcode, int vlen, BasicType bt);
+
   // identify extra cases that we might want to provide match rules for
   // e.g. Op_ vector nodes and other intrinsics while guarding with vlen
   static const bool match_rule_supported_vector(int opcode, int vlen, BasicType bt);

src/hotspot/share/opto/superword.cpp

@@ -374,8 +374,10 @@ void SuperWord::unrolling_analysis(int &local_loop_unroll_factor) {
         break;
       }
 
-      // Map the maximal common vector
-      if (VectorNode::implemented(n->Opcode(), cur_max_vector, bt)) {
+      // Map the maximal common vector except conversion nodes, because we can't get
+      // the precise basic type for conversion nodes in the stage of early analysis.
+      if (!VectorNode::is_convert_opcode(n->Opcode()) &&
+          VectorNode::implemented(n->Opcode(), cur_max_vector, bt)) {
         if (cur_max_vector < max_vector && !flag_small_bt) {
           max_vector = cur_max_vector;
         } else if (cur_max_vector > max_vector && UseSubwordForMaxVector) {
@@ -1005,6 +1007,12 @@ int SuperWord::get_vw_bytes_special(MemNode* s) {
     }
   }
 
+  // Check for special case where there is a type conversion between different data size.
+  int vectsize = max_vector_size_in_def_use_chain(s);
+  if (vectsize < max_vector_size(btype)) {
+    vw = MIN2(vectsize * type2aelembytes(btype), vw);
+  }
+
   return vw;
 }
 
@@ -1193,7 +1201,9 @@ bool SuperWord::stmts_can_pack(Node* s1, Node* s2, int align) {
   BasicType bt2 = velt_basic_type(s2);
   if(!is_java_primitive(bt1) || !is_java_primitive(bt2))
     return false;
-  if (Matcher::max_vector_size(bt1) < 2) {
+  BasicType longer_bt = longer_type_for_conversion(s1);
+  if (max_vector_size(bt1) < 2 ||
+      (longer_bt != T_ILLEGAL && max_vector_size(longer_bt) < 2)) {
     return false; // No vectors for this type
   }
 
@@ -1436,6 +1446,16 @@ void SuperWord::extend_packlist() {
   }
 }
 
+//------------------------------adjust_alignment_for_type_conversion---------------------------------
+// Adjust the target alignment if conversion between different data size exists in def-use nodes.
+int SuperWord::adjust_alignment_for_type_conversion(Node* s, Node* t, int align) {
+  if (longer_type_for_conversion(s) != T_ILLEGAL ||
+      longer_type_for_conversion(t) != T_ILLEGAL) {
+    align = align / data_size(s) * data_size(t);
+  }
+  return align;
+}
+
 //------------------------------follow_use_defs---------------------------
 // Extend the packset by visiting operand definitions of nodes in pack p
 bool SuperWord::follow_use_defs(Node_List* p) {
@@ -1447,16 +1467,17 @@ bool SuperWord::follow_use_defs(Node_List* p) {
 
   if (s1->is_Load()) return false;
 
-  int align = alignment(s1);
-  NOT_PRODUCT(if(is_trace_alignment()) tty->print_cr("SuperWord::follow_use_defs: s1 %d, align %d", s1->_idx, align);)
+  NOT_PRODUCT(if(is_trace_alignment()) tty->print_cr("SuperWord::follow_use_defs: s1 %d, align %d", s1->_idx, alignment(s1));)
   bool changed = false;
   int start = s1->is_Store() ? MemNode::ValueIn   : 1;
   int end   = s1->is_Store() ? MemNode::ValueIn+1 : s1->req();
   for (int j = start; j < end; j++) {
+    int align = alignment(s1);
     Node* t1 = s1->in(j);
     Node* t2 = s2->in(j);
     if (!in_bb(t1) || !in_bb(t2))
       continue;
+    align = adjust_alignment_for_type_conversion(s1, t1, align);
     if (stmts_can_pack(t1, t2, align)) {
       if (est_savings(t1, t2) >= 0) {
         Node_List* pair = new Node_List();
@@ -1500,12 +1521,15 @@ bool SuperWord::follow_def_uses(Node_List* p) {
       if (t2->Opcode() == Op_AddI && t2 == _lp->as_CountedLoop()->incr()) continue; // don't mess with the iv
       if (!opnd_positions_match(s1, t1, s2, t2))
         continue;
-      if (stmts_can_pack(t1, t2, align)) {
+      int adjusted_align = alignment(s1);
+      adjusted_align = adjust_alignment_for_type_conversion(s1, t1, adjusted_align);
+      if (stmts_can_pack(t1, t2, adjusted_align)) {
         int my_savings = est_savings(t1, t2);
         if (my_savings > savings) {
           savings = my_savings;
           u1 = t1;
           u2 = t2;
+          align = adjusted_align;
         }
       }
     }
@@ -1698,8 +1722,7 @@ void SuperWord::combine_packs() {
   for (int i = 0; i < _packset.length(); i++) {
     Node_List* p1 = _packset.at(i);
     if (p1 != NULL) {
-      BasicType bt = velt_basic_type(p1->at(0));
-      uint max_vlen = max_vector_size(bt); // Max elements in vector
+      uint max_vlen = max_vector_size_in_def_use_chain(p1->at(0)); // Max elements in vector
       assert(is_power_of_2(max_vlen), "sanity");
       uint psize = p1->size();
       if (!is_power_of_2(psize)) {
@@ -2022,6 +2045,8 @@ bool SuperWord::implemented(Node_List* p) {
       } else {
         retValue = ReductionNode::implemented(opc, size, arith_type->basic_type());
       }
+    } else if (VectorNode::is_convert_opcode(opc)) {
+      retValue = VectorCastNode::implemented(opc, size, velt_basic_type(p0->in(1)), velt_basic_type(p0));
     } else {
       // Vector unsigned right shift for signed subword types behaves differently
       // from Java Spec. But when the shift amount is a constant not greater than
@@ -2616,12 +2641,11 @@ bool SuperWord::output() {
         Node* in = vector_opd(p, 1);
         vn = VectorNode::make(opc, in, NULL, vlen, velt_basic_type(n));
         vlen_in_bytes = vn->as_Vector()->length_in_bytes();
-      } else if (opc == Op_ConvI2F || opc == Op_ConvL2D ||
-                 opc == Op_ConvF2I || opc == Op_ConvD2L) {
+      } else if (VectorNode::is_convert_opcode(opc)) {
         assert(n->req() == 2, "only one input expected");
         BasicType bt = velt_basic_type(n);
-        int vopc = VectorNode::opcode(opc, bt);
         Node* in = vector_opd(p, 1);
+        int vopc = VectorCastNode::opcode(in->bottom_type()->is_vect()->element_basic_type());
         vn = VectorCastNode::make(vopc, in, bt, vlen);
         vlen_in_bytes = vn->as_Vector()->length_in_bytes();
       } else if (is_cmov_pack(p)) {
@@ -3134,9 +3158,26 @@ bool SuperWord::is_vector_use(Node* use, int u_idx) {
     return true;
   }
 
-
   if (u_pk->size() != d_pk->size())
     return false;
+
+  if (longer_type_for_conversion(use) != T_ILLEGAL) {
+    // type conversion takes a type of a kind of size and produces a type of
+    // another size - hence the special checks on alignment and size.
+    for (uint i = 0; i < u_pk->size(); i++) {
+      Node* ui = u_pk->at(i);
+      Node* di = d_pk->at(i);
+      if (ui->in(u_idx) != di) {
+        return false;
+      }
+      if (alignment(ui) / type2aelembytes(velt_basic_type(ui)) !=
+          alignment(di) / type2aelembytes(velt_basic_type(di))) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   for (uint i = 0; i < u_pk->size(); i++) {
     Node* ui = u_pk->at(i);
     Node* di = d_pk->at(i);
@@ -3369,6 +3410,63 @@ void SuperWord::compute_max_depth() {
   }
 }
 
+BasicType SuperWord::longer_type_for_conversion(Node* n) {
+  int opcode = n->Opcode();
+  switch (opcode) {
+    case Op_ConvD2I:
+    case Op_ConvI2D:
+    case Op_ConvF2D:
+    case Op_ConvD2F: return T_DOUBLE;
+    case Op_ConvF2L:
+    case Op_ConvL2F:
+    case Op_ConvL2I:
+    case Op_ConvI2L: return T_LONG;
+    case Op_ConvI2F: {
+      BasicType src_t = velt_basic_type(n->in(1));
+      if (src_t == T_BYTE || src_t == T_SHORT) {
+        return T_FLOAT;
+      }
+      return T_ILLEGAL;
+    }
+    case Op_ConvF2I: {
+      BasicType dst_t = velt_basic_type(n);
+      if (dst_t == T_BYTE || dst_t == T_SHORT) {
+        return T_FLOAT;
+      }
+      return T_ILLEGAL;
+    }
+  }
+  return T_ILLEGAL;
+}
+
+int SuperWord::max_vector_size_in_def_use_chain(Node* n) {
+  BasicType bt = velt_basic_type(n);
+  BasicType vt = bt;
+
+  // find the longest type among def nodes.
+  uint start, end;
+  VectorNode::vector_operands(n, &start, &end);
+  for (uint i = start; i < end; ++i) {
+    Node* input = n->in(i);
+    if (!in_bb(input)) continue;
+    BasicType newt = longer_type_for_conversion(input);
+    vt = (newt == T_ILLEGAL) ? vt : newt;
+  }
+
+  // find the longest type among use nodes.
+  for (uint i = 0; i < n->outcnt(); ++i) {
+    Node* output = n->raw_out(i);
+    if (!in_bb(output)) continue;
+    BasicType newt = longer_type_for_conversion(output);
+    vt = (newt == T_ILLEGAL) ? vt : newt;
+  }
+
+  int max = max_vector_size(vt);
+  // If now there is no vectors for the longest type, the nodes with the longest
+  // type in the def-use chain are not packed in SuperWord::stmts_can_pack.
+  return max < 2 ? max_vector_size(bt) : max;
+}
+
 //-------------------------compute_vector_element_type-----------------------
 // Compute necessary vector element type for expressions
 // This propagates backwards a narrower integer type when the

src/hotspot/share/opto/superword.hpp

@@ -518,6 +518,7 @@ class SuperWord : public ResourceObj {
   int data_size(Node* s);
   // Extend packset by following use->def and def->use links from pack members.
   void extend_packlist();
+  int adjust_alignment_for_type_conversion(Node* s, Node* t, int align);
   // Extend the packset by visiting operand definitions of nodes in pack p
   bool follow_use_defs(Node_List* p);
   // Extend the packset by visiting uses of nodes in pack p
@@ -571,6 +572,10 @@ class SuperWord : public ResourceObj {
   void bb_insert_after(Node* n, int pos);
   // Compute max depth for expressions from beginning of block
   void compute_max_depth();
+  // Return the longer type for type-conversion node and return illegal type for other nodes.
+  BasicType longer_type_for_conversion(Node* n);
+  // Find the longest type in def-use chain for packed nodes, and then compute the max vector size.
+  int max_vector_size_in_def_use_chain(Node* n);
   // Compute necessary vector element type for expressions
   void compute_vector_element_type();
   // Are s1 and s2 in a pack pair and ordered as s1,s2?