From 1f3c5899acaf6ac6bb09d2497b6b1b78ba3a92b3 Mon Sep 17 00:00:00 2001 From: erfang Date: Wed, 19 Mar 2025 02:36:01 +0000 Subject: [PATCH 01/12] 8354242: VectorAPI: combine vector not operation with compare This patch optimizes the following patterns: For integer types: ``` (XorV (VectorMaskCmp src1 src2 cond) (Replicate -1)) => (VectorMaskCmp src1 src2 ncond) (XorVMask (VectorMaskCmp src1 src2 cond) (MaskAll m1)) => (VectorMaskCmp src1 src2 ncond) ``` cond can be eq, ne, le, ge, lt, gt, ule, uge, ult and ugt, ncond is the negative comparison of cond. For float and double types: ``` (XorV (VectorMaskCast (VectorMaskCmp src1 src2 cond)) (Replicate -1)) => (VectorMaskCast (VectorMaskCmp src1 src2 ncond)) (XorVMask (VectorMaskCast (VectorMaskCmp src1 src2 cond)) (MaskAll m1)) => (VectorMaskCast (VectorMaskCmp src1 src2 ncond)) ``` cond can be eq or ne. Benchmarks on Nvidia Grace machine with 128-bit SVE2: With option `-XX:UseSVE=2`: ``` Benchmark Unit Before Score Error After Score Error Uplift testCompareEQMaskNotByte ops/s 7912127.225 2677.289518 10266136.26 8955.008548 1.29 testCompareEQMaskNotDouble ops/s 884737.6799 446.963779 1179760.772 448.031844 1.33 testCompareEQMaskNotFloat ops/s 1765045.787 682.332214 2359520.803 896.305743 1.33 testCompareEQMaskNotInt ops/s 1787221.411 977.743935 2353952.519 960.069976 1.31 testCompareEQMaskNotLong ops/s 895297.1974 673.44808 1178449.02 323.804205 1.31 testCompareEQMaskNotShort ops/s 3339987.002 3415.2226 4712761.965 2110.862053 1.41 testCompareGEMaskNotByte ops/s 7907615.16 4094.243652 10251646.9 9486.699831 1.29 testCompareGEMaskNotInt ops/s 1683738.958 4233.813092 2352855.205 1251.952546 1.39 testCompareGEMaskNotLong ops/s 854496.1561 8594.598885 1177811.493 521.1229 1.37 testCompareGEMaskNotShort ops/s 3341860.309 1578.975338 4714008.434 1681.10365 1.41 testCompareGTMaskNotByte ops/s 7910823.674 2993.367032 10245063.58 9774.75138 1.29 testCompareGTMaskNotInt ops/s 1673393.928 3153.099431 2353654.521 1190.848583 1.4 testCompareGTMaskNotLong ops/s 849405.9159 2432.858159 1177952.041 359.96413 1.38 testCompareGTMaskNotShort ops/s 3339509.141 3339.976585 4711442.496 2673.364893 1.41 testCompareLEMaskNotByte ops/s 7911340.004 3114.69191 10231626.5 27134.20035 1.29 testCompareLEMaskNotInt ops/s 1675812.113 1340.969885 2353255.341 1452.4522 1.4 testCompareLEMaskNotLong ops/s 848862.8036 6564.841731 1177763.623 539.290106 1.38 testCompareLEMaskNotShort ops/s 3324951.54 2380.29473 4712116.251 1544.559684 1.41 testCompareLTMaskNotByte ops/s 7910390.844 2630.861436 10239567.69 6487.441672 1.29 testCompareLTMaskNotInt ops/s 1672180.09 995.238142 2353757.863 853.774734 1.4 testCompareLTMaskNotLong ops/s 856502.2695 12276.82851 1177671.815 496.723302 1.37 testCompareLTMaskNotShort ops/s 3325798.025 2412.702501 4711554.181 1779.302112 1.41 testCompareNEMaskNotByte ops/s 7910002.518 2771.82477 10245315.33 16321.93935 1.29 testCompareNEMaskNotDouble ops/s 863754.6022 523.140788 1179133.982 476.572178 1.36 testCompareNEMaskNotFloat ops/s 1723321.883 2598.484803 2358492.186 877.1401 1.36 testCompareNEMaskNotInt ops/s 1670288.841 751.774826 2354158.125 835.720163 1.4 testCompareNEMaskNotLong ops/s 836327.6835 410.525466 1178178.825 308.757932 1.4 testCompareNEMaskNotShort ops/s 3327815.841 1511.978763 4711379.136 2336.505531 1.41 testCompareUGEMaskNotByte ops/s 7906699.024 3200.936474 10253843.74 15067.59401 1.29 testCompareUGEMaskNotInt ops/s 1674003.923 3287.191727 2353340.666 951.381021 1.4 testCompareUGEMaskNotLong ops/s 852424.5562 8920.408939 1177943.609 389.6621 1.38 testCompareUGEMaskNotShort ops/s 3327255.858 1584.885143 4711622.355 1247.215277 1.41 testCompareUGTMaskNotByte ops/s 7909249.189 4435.283667 10245541.34 10993.34739 1.29 testCompareUGTMaskNotInt ops/s 1693713.433 20650.00213 2353153.787 1055.343846 1.38 testCompareUGTMaskNotLong ops/s 851022.3395 7079.065268 1177910.677 538.604598 1.38 testCompareUGTMaskNotShort ops/s 3327236.988 1616.886789 4711209.865 3098.494145 1.41 testCompareULEMaskNotByte ops/s 7909350.825 3251.262342 10261449.03 7273.831341 1.29 testCompareULEMaskNotInt ops/s 1672350.925 1545.304304 2353231.755 914.231193 1.4 testCompareULEMaskNotLong ops/s 853349.4765 9804.906913 1177967.254 435.044367 1.38 testCompareULEMaskNotShort ops/s 3325757.891 1555.062257 4712873.187 1650.986905 1.41 testCompareULTMaskNotByte ops/s 7912218.621 2633.477744 10242095.98 21921.39902 1.29 testCompareULTMaskNotInt ops/s 1673994.849 2672.507666 2353449.22 946.105757 1.4 testCompareULTMaskNotLong ops/s 849032.5868 10406.06689 1177586.047 506.541456 1.38 testCompareULTMaskNotShort ops/s 3328062.026 1892.991844 4713247.216 1855.983724 1.41 ``` With option `-XX:UseSVE=0`: ``` Benchmark Unit Before Score Error After Score Error Uplift testCompareEQMaskNotByte ops/s 7895961.919 72712.90804 7746493.731 71481.92938 0.98 testCompareEQMaskNotDouble ops/s 789811.0455 384.493088 766473.7994 2216.581793 0.97 testCompareEQMaskNotFloat ops/s 1806305.818 638.010451 1819616.613 3295.38958 1 testCompareEQMaskNotInt ops/s 1815820.144 1225.336135 1849538.401 766.29902 1.01 testCompareEQMaskNotLong ops/s 807336.492 335.451807 792732.9483 277.954432 0.98 testCompareEQMaskNotShort ops/s 4818266.38 1927.862665 4668903.001 1922.782715 0.96 testCompareGEMaskNotByte ops/s 7818439.678 75374.97739 16498003.98 41440.49653 2.11 testCompareGEMaskNotInt ops/s 1815159.05 1090.912209 2372095.779 1664.397112 1.3 testCompareGEMaskNotLong ops/s 804324.5575 2301.686878 927919.8507 371.766719 1.15 testCompareGEMaskNotShort ops/s 4818966.563 2443.643652 5385561.038 29558.37423 1.11 testCompareGTMaskNotByte ops/s 7893406.157 82687.74264 16470663.2 22165.55812 2.08 testCompareGTMaskNotInt ops/s 1815316.812 915.894106 2370447.198 655.016338 1.3 testCompareGTMaskNotLong ops/s 807019.456 526.525482 928079.0541 330.582693 1.15 testCompareGTMaskNotShort ops/s 4820552.881 1684.247747 5355902.93 5893.2915 1.11 testCompareLEMaskNotByte ops/s 7816263.323 79560.0015 16473621.19 56688.99585 2.1 testCompareLEMaskNotInt ops/s 1814915.724 926.998625 2368790.306 932.594778 1.3 testCompareLEMaskNotLong ops/s 806483.9 935.718082 928110.9074 407.096695 1.15 testCompareLEMaskNotShort ops/s 4813660.241 6817.870509 5357107.852 10061.47975 1.11 testCompareLTMaskNotByte ops/s 7838948.962 69136.4504 16424405.96 24464.75469 2.09 testCompareLTMaskNotInt ops/s 1815056.833 1187.6453 2369892.187 1103.819634 1.3 testCompareLTMaskNotLong ops/s 806602.1804 287.923365 928346.4118 617.682824 1.15 testCompareLTMaskNotShort ops/s 4817940.643 2767.1509 5372537.84 15397.47169 1.11 testCompareNEMaskNotByte ops/s 9078493.798 4630.339307 16484348.42 18925.88346 1.81 testCompareNEMaskNotDouble ops/s 661769.6272 398.712981 926763.5839 1808.843788 1.4 testCompareNEMaskNotFloat ops/s 1570527.252 563.642144 2312425.678 1815.844846 1.47 testCompareNEMaskNotInt ops/s 1619146.58 626.793854 2369711.543 942.330478 1.46 testCompareNEMaskNotLong ops/s 680201.5381 2252.836482 927808.6147 414.917863 1.36 testCompareNEMaskNotShort ops/s 3763508.054 3622.560798 5367808.015 8591.466599 1.42 testCompareUGEMaskNotByte ops/s 7886373.129 75917.74675 16480928.93 27524.31005 2.08 testCompareUGEMaskNotInt ops/s 1815636.832 750.036241 2369683.015 901.609404 1.3 testCompareUGEMaskNotLong ops/s 806862.5826 287.819616 928001.4394 361.063837 1.15 testCompareUGEMaskNotShort ops/s 4820581.361 2098.537435 5375854.248 25619.40165 1.11 testCompareUGTMaskNotByte ops/s 7891591.465 96614.93542 16410405.93 15012.37096 2.07 testCompareUGTMaskNotInt ops/s 1814871.179 662.825588 2371325.903 1170.491164 1.3 testCompareUGTMaskNotLong ops/s 804013.7658 2240.534209 928062.2169 531.306897 1.15 testCompareUGTMaskNotShort ops/s 4818150.337 3051.717685 5381449.337 21212.34187 1.11 testCompareULEMaskNotByte ops/s 7831540.628 81306.67253 16495250.78 38682.19675 2.1 testCompareULEMaskNotInt ops/s 1814484.14 687.860656 2369265.075 940.609586 1.3 testCompareULEMaskNotLong ops/s 807780.5749 769.876816 927538.0732 1278.267724 1.14 testCompareULEMaskNotShort ops/s 4817437.42 5141.336541 5356183.359 7015.608124 1.11 testCompareULTMaskNotByte ops/s 7849078.225 56753.59764 16395975.27 34043.67295 2.08 testCompareULTMaskNotInt ops/s 1814328.226 2697.219111 2370700.47 1991.841988 1.3 testCompareULTMaskNotLong ops/s 807166.8197 253.061506 927926.2803 252.933462 1.14 testCompareULTMaskNotShort ops/s 4821098.216 1625.959044 5348980.243 4100.768121 1.1 ``` Benchmarks on AMD EPYC 9124 16-Core Processor: With option `-XX:UseAVX=3`: ``` Benchmark Unit Before Score Error After Score Error Uplift testCompareEQMaskNotByte ops/s 16607323.35 1233692.631 18381557.66 1163201.522 1.1 testCompareEQMaskNotDouble ops/s 2114285.245 58782.2534 2959946.353 43016.0445 1.39 testCompareEQMaskNotFloat ops/s 4480874.437 89975.29074 6960151.436 64799.143 1.55 testCompareEQMaskNotInt ops/s 4370906.91 51784.80889 6856955.043 313858.5504 1.56 testCompareEQMaskNotLong ops/s 2080065.895 26762.06732 2939142.143 67179.05314 1.41 testCompareEQMaskNotShort ops/s 7968282.563 210437.2781 12701214.56 473152.6407 1.59 testCompareGEMaskNotByte ops/s 18419141.89 473408.9451 19880059.68 321638.0397 1.07 testCompareGEMaskNotInt ops/s 4419015.62 77352.98633 7037639.227 151066.0383 1.59 testCompareGEMaskNotLong ops/s 2147982.48 49227.42782 3000275.928 39298.75344 1.39 testCompareGEMaskNotShort ops/s 8469039.613 17833.19707 12288229.49 244317.8812 1.45 testCompareGTMaskNotByte ops/s 18728997.5 468328.8358 20544730.05 392264.6466 1.09 testCompareGTMaskNotInt ops/s 4510009.705 78812.57357 7364629.942 70970.78473 1.63 testCompareGTMaskNotLong ops/s 2124104.969 40917.89257 2953536.279 35199.19687 1.39 testCompareGTMaskNotShort ops/s 8690557.621 311534.1159 12344017.51 457931.8741 1.42 testCompareLEMaskNotByte ops/s 17758400.53 478383.4945 19209183.26 1143297.241 1.08 testCompareLEMaskNotInt ops/s 4363664.862 43443.18063 7054093.064 78141.11476 1.61 testCompareLEMaskNotLong ops/s 2068632.213 29844.78023 2954766.412 50667.22502 1.42 testCompareLEMaskNotShort ops/s 8637608.548 183538.5511 12719010.27 473568.8825 1.47 testCompareLTMaskNotByte ops/s 14406138.95 423105.0163 17292417.96 371386.9689 1.2 testCompareLTMaskNotInt ops/s 4546707.266 131977.3144 7040483.394 213590.4657 1.54 testCompareLTMaskNotLong ops/s 2123277.356 47243.21499 2848720.442 58896.97045 1.34 testCompareLTMaskNotShort ops/s 7570169.363 649873.6295 11945383.75 988276.5955 1.57 testCompareNEMaskNotByte ops/s 18274529.55 683396.7384 19081938.8 1118739.778 1.04 testCompareNEMaskNotDouble ops/s 2112533.61 43295.50012 2912115.441 78189.51083 1.37 testCompareNEMaskNotFloat ops/s 4628683.814 93817.07362 6967208.729 145135.8544 1.5 testCompareNEMaskNotInt ops/s 4470900.214 75974.50842 7286913.662 116328.5277 1.62 testCompareNEMaskNotLong ops/s 2134091.061 46377.94061 2934667.477 81675.46021 1.37 testCompareNEMaskNotShort ops/s 8790384.287 396161.8599 13076858.35 286272.1155 1.48 testCompareUGEMaskNotByte ops/s 18009150.9 660803.8886 17551258.33 1667014.843 0.97 testCompareUGEMaskNotInt ops/s 4442928.74 83190.81019 6854088.277 329008.8901 1.54 testCompareUGEMaskNotLong ops/s 2088357.736 71696.24791 2973202.26 63278.78974 1.42 testCompareUGEMaskNotShort ops/s 8348624.02 116562.7876 12832250.78 546869.3006 1.53 testCompareUGTMaskNotByte ops/s 17871101.25 800199.6321 19902619.81 214003.3262 1.11 testCompareUGTMaskNotInt ops/s 4088304.421 137797.9723 7135454.33 124553.651 1.74 testCompareUGTMaskNotLong ops/s 2070610.42 19881.82182 2991536.365 36260.60767 1.44 testCompareUGTMaskNotShort ops/s 8637099.341 155822.1608 12756579.77 186068.199 1.47 testCompareULEMaskNotByte ops/s 17940901.36 1258029.364 18932484.94 694554.6305 1.05 testCompareULEMaskNotInt ops/s 4369177.511 74982.31936 6392773.082 550171.2266 1.46 testCompareULEMaskNotLong ops/s 2135905.761 43693.63178 2877579.631 41651.56289 1.34 testCompareULEMaskNotShort ops/s 8607710.544 132655.1676 12446370.04 441718.3035 1.44 testCompareULTMaskNotByte ops/s 17409912.23 1033204.537 20607479.99 362000.5056 1.18 testCompareULTMaskNotInt ops/s 4386455.9 119192.1635 6920123.264 186158.2845 1.57 testCompareULTMaskNotLong ops/s 2064995.149 38622.2734 2988343.589 39037.90006 1.44 testCompareULTMaskNotShort ops/s 8642182.752 230919.2442 13029582.09 437101.4923 1.5 ``` The small amount of performance degradation is due to test fluctuations. --- src/hotspot/share/opto/node.cpp | 8 + src/hotspot/share/opto/vectornode.cpp | 91 ++- src/hotspot/share/opto/vectornode.hpp | 1 + .../vectorapi/VectorMaskCompareNotTest.java | 578 ++++++++++++++++++ .../vector/MaskCompareNotBenchmark.java | 376 ++++++++++++ 5 files changed, 1051 insertions(+), 3 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java create mode 100644 test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index d00bd3d394371..0c4ab226aa622 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -1216,6 +1216,14 @@ bool Node::has_special_unique_user() const { } else if ((is_IfFalse() || is_IfTrue()) && n->is_If()) { // See IfNode::fold_compares return true; + } else if (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) { + // Condition for removing the unnecessary not() following compare(...) operation. + // The predecessor of XorV/XorVMask may be used by an useless VectorBox node, + // making it's not single used by XorV/XorVMask. The VectorBox node will be + // eliminated by the phase RemoveUseless, but the XorV/XorVMask node may not + // be added to the IGVN worklist, then the optimization will not be applied. + // Therefore, add this node into IGVN worklist to make the optimization happen. + return true; } else { return false; } diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 084b70a690653..17fa86662acf0 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2164,7 +2164,36 @@ Node* OrVNode::Identity(PhaseGVN* phase) { return redundant_logical_identity(this); } +// Returns whether (XorV (VectorMaskCmp) -1) can be optimized by performing the +// inverse of a comparison operation. +bool VectorMaskCmpNode::predicate_can_be_inverted() { + switch (_predicate) { + case BoolTest::eq: + case BoolTest::ne: + // eq and ne also apply to floating-point special values like NaN and infinities. + return true; + case BoolTest::le: + case BoolTest::ge: + case BoolTest::lt: + case BoolTest::gt: + case BoolTest::ule: + case BoolTest::uge: + case BoolTest::ult: + case BoolTest::ugt: { + BasicType bt = vect_type()->element_basic_type(); + // For float and double, we don't know if either comparison operand is a + // NaN, NaN {le|ge|lt|gt} anything is false, resulting in inconsistent + // results before and after negation. + return is_integral_type(bt); + } + default: + return false; + } +} + Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { + Node* in1 = in(1); + Node* in2 = in(2); // (XorV src src) => (Replicate zero) // (XorVMask src src) => (MaskAll zero) // @@ -2172,12 +2201,68 @@ Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { // operation, since the VectorAPI masked operation requires // the unmasked lanes to save the same values in the first // operand. - if (!is_predicated_vector() && (in(1) == in(2))) { + if (!is_predicated_vector() && (in1 == in2)) { BasicType bt = vect_type()->element_basic_type(); Node* zero = phase->transform(phase->zerocon(bt)); - return VectorNode::scalar2vector(zero, length(), bt, bottom_type()->isa_vectmask() != nullptr); + return VectorNode::scalar2vector(zero, length(), bt, + bottom_type()->isa_vectmask() != nullptr); + } + + // Transformations for predicated IRs are not supported for now. + if (is_predicated_vector() || in1->is_predicated_vector() || + in2->is_predicated_vector()) { + return VectorNode::Ideal(phase, can_reshape); } - return VectorNode::Ideal(phase, can_reshape); + + // For integer types: + // (XorV (VectorMaskCmp src1 src2 cond) (Replicate -1)) + // => (VectorMaskCmp src1 src2 ncond) + // (XorVMask (VectorMaskCmp src1 src2 cond) (MaskAll m1)) + // => (VectorMaskCmp src1 src2 ncond) + // cond can be eq, ne, le, ge, lt, gt, ule, uge, ult and ugt, ncond is the + // negative comparison of cond. + // + // For float and double types: + // (XorV (VectorMaskCast (VectorMaskCmp src1 src2 cond)) (Replicate -1)) + // => (VectorMaskCast (VectorMaskCmp src1 src2 ncond)) + // (XorVMask (VectorMaskCast (VectorMaskCmp src1 src2 cond)) (MaskAll m1)) + // => (VectorMaskCast (VectorMaskCmp src1 src2 ncond)) + // cond can be eq or ne. + // + // XorV/XorVMask is commutative, swap VectorMaskCmp/Op_VectorMaskCast to in1. + if (in1->Opcode() != Op_VectorMaskCmp && in1->Opcode() != Op_VectorMaskCast) { + swap(in1, in2); + } + + const TypeVect* vmcast_vt = nullptr; + if (in1->Opcode() == Op_VectorMaskCast && in1->outcnt() == 1 && + in1->in(1)->Opcode() == Op_VectorMaskCmp) { + vmcast_vt = in1->as_Vector()->vect_type(); + in1 = in1->in(1); + } + if (in1->Opcode() != Op_VectorMaskCmp || in1->outcnt() > 1 || + !((VectorMaskCmpNode*) in1)->predicate_can_be_inverted() || + !VectorNode::is_all_ones_vector(in2)) { + return VectorNode::Ideal(phase, can_reshape); + } + + // This is the same with BoolTest::negate(), but we can't call it with a + // BoolTest object because the comparison may be unsigned comparison, but + // BoolTest doesn't support unsigned comparisons. + BoolTest::mask neg_cond = + (BoolTest::mask) (((VectorMaskCmpNode*) in1)->get_predicate() ^ 4); + + ConINode* predicate_node = phase->intcon(neg_cond); + const TypeVect* vt = in1->as_Vector()->vect_type(); + Node* vmcmp = new VectorMaskCmpNode(neg_cond, in1->in(1), in1->in(2), + predicate_node, vt); + if (vmcast_vt != nullptr) { + // We optimized out an VectorMaskCast, and in order to ensure type + // correctness, we need to regenerate one. VectorMaskCast will be encoded as + // empty for types with the same size. + vmcmp = new VectorMaskCastNode(phase->transform(vmcmp), vmcast_vt); + } + return vmcmp; } Node* VectorBlendNode::Identity(PhaseGVN* phase) { diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 36706a7b7a14b..b55aac26f041e 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1679,6 +1679,7 @@ class VectorMaskCmpNode : public VectorNode { virtual bool cmp( const Node &n ) const { return VectorNode::cmp(n) && _predicate == ((VectorMaskCmpNode&)n)._predicate; } + bool predicate_can_be_inverted(); BoolTest::mask get_predicate() { return _predicate; } #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java new file mode 100644 index 0000000000000..10c729128032d --- /dev/null +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java @@ -0,0 +1,578 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package compiler.vectorapi; + +import compiler.lib.ir_framework.*; +import compiler.lib.generators.*; +import jdk.incubator.vector.*; +import jdk.test.lib.Asserts; + +/* + * @test + * @bug 8354242 + * @key randomness + * @library /test/lib / + * @summary test combining vector not operation with compare + * @modules jdk.incubator.vector + * @requires ((os.arch!="x86" & os.arch!="i386" & os.arch!="amd64" & os.arch!="x86_64") | vm.cpu.features ~= ".*avx.*") + * + * @run driver compiler.vectorapi.VectorMaskCompareNotTest + */ + +public class VectorMaskCompareNotTest { + private static int LENGTH = 128; + private static final VectorSpecies B_SPECIES = ByteVector.SPECIES_MAX; + private static final VectorSpecies S_SPECIES = ShortVector.SPECIES_MAX; + private static final VectorSpecies I_SPECIES = IntVector.SPECIES_MAX; + private static final VectorSpecies L_SPECIES = LongVector.SPECIES_MAX; + private static final VectorSpecies F_SPECIES = FloatVector.SPECIES_MAX; + private static final VectorSpecies D_SPECIES = DoubleVector.SPECIES_MAX; + private static final Generators RD = Generators.G; + + private static byte[] ba; + private static byte[] bb; + private static short[] sa; + private static short[] sb; + private static int[] ia; + private static int[] ib; + private static long[] la; + private static long[] lb; + private static float[] fa; + private static float[] fb; + private static float[] fnan; + private static float[] fpinf; + private static float[] fninf; + private static double[] da; + private static double[] db; + private static double[] dnan; + private static double[] dpinf; + private static double[] dninf; + private static boolean[] mr; + + static { + ba = new byte[LENGTH]; + bb = new byte[LENGTH]; + sa = new short[LENGTH]; + sb = new short[LENGTH]; + ia = new int[LENGTH]; + ib = new int[LENGTH]; + la = new long[LENGTH]; + lb = new long[LENGTH]; + fa = new float[LENGTH]; + fb = new float[LENGTH]; + fnan = new float[LENGTH]; + fpinf = new float[LENGTH]; + fninf = new float[LENGTH]; + da = new double[LENGTH]; + db = new double[LENGTH]; + dnan = new double[LENGTH]; + dpinf = new double[LENGTH]; + dninf = new double[LENGTH]; + mr = new boolean[LENGTH]; + + Generator iGen = RD.uniformInts(Integer.MIN_VALUE, Integer.MAX_VALUE); + Generator lGen = RD.uniformLongs(Long.MIN_VALUE, Long.MAX_VALUE); + Generator fGen = RD.uniformFloats(Float.MIN_VALUE, Float.MAX_VALUE); + Generator dGen = RD.uniformDoubles(Double.MIN_VALUE, Double.MAX_VALUE); + for (int i = 0; i < LENGTH; i++) { + ba[i] = iGen.next().byteValue(); + bb[i] = iGen.next().byteValue(); + sa[i] = iGen.next().shortValue(); + sb[i] = iGen.next().shortValue(); + ia[i] = iGen.next(); + ib[i] = iGen.next(); + la[i] = lGen.next(); + lb[i] = lGen.next(); + fa[i] = fGen.next(); + fb[i] = fGen.next(); + fnan[i] = Float.NaN; + fpinf[i] = Float.POSITIVE_INFINITY; + fninf[i] = Float.NEGATIVE_INFINITY; + da[i] = dGen.next(); + db[i] = dGen.next(); + dnan[i] = Double.NaN; + dpinf[i] = Double.POSITIVE_INFINITY; + dninf[i] = Double.NEGATIVE_INFINITY; + } + } + + public static int compareUnsigned(Number a, Number b) { + if (a instanceof Byte) { + return Integer.compareUnsigned(Byte.toUnsignedInt(a.byteValue()), + Byte.toUnsignedInt(b.byteValue())); + } else if (a instanceof Short) { + return Integer.compareUnsigned(Short.toUnsignedInt(a.shortValue()), + Short.toUnsignedInt(b.shortValue())); + } else if (a instanceof Integer) { + return Integer.compareUnsigned(a.intValue(), b.intValue()); + } else if (a instanceof Long) { + return Long.compareUnsigned(a.longValue(), b.longValue()); + } + return 0; + } + + public static > void verifyResults(T a, T b, boolean r, + VectorOperators.Comparison op) { + if (op == VectorOperators.EQ) { + Asserts.assertEquals(a.compareTo(b) != 0, r); + } else if (op == VectorOperators.NE) { + Asserts.assertEquals(a.compareTo(b) == 0, r); + } else if (op == VectorOperators.LE) { + Asserts.assertEquals(a.compareTo(b) > 0, r); + } else if (op == VectorOperators.GE) { + Asserts.assertEquals(a.compareTo(b) < 0, r); + } else if (op == VectorOperators.LT) { + Asserts.assertEquals(a.compareTo(b) >= 0, r); + } else if (op == VectorOperators.GT) { + Asserts.assertEquals(a.compareTo(b) <= 0, r); + } else if (op == VectorOperators.ULE) { + Asserts.assertEquals(compareUnsigned(a, b) > 0, r); + } else if (op == VectorOperators.UGE) { + Asserts.assertEquals(compareUnsigned(a, b) < 0, r); + } else if (op == VectorOperators.ULT) { + Asserts.assertEquals(compareUnsigned(a, b) >= 0, r); + } else if (op == VectorOperators.UGT) { + Asserts.assertEquals(compareUnsigned(a, b) <= 0, r); + } + } + + @ForceInline + public static void testCompareMaskNotByte(VectorOperators.Comparison op) { + ByteVector av = ByteVector.fromArray(B_SPECIES, ba, 0); + ByteVector bv = ByteVector.fromArray(B_SPECIES, bb, 0); + VectorMask m1 = av.compare(op, bv); + m1.not().intoArray(mr, 0); + + for (int i = 0; i < B_SPECIES.length(); i++) { + verifyResults(ba[i], bb[i], mr[i], op); + } + } + + @ForceInline + public static void testCompareMaskNotShort(VectorOperators.Comparison op) { + ShortVector av = ShortVector.fromArray(S_SPECIES, sa, 0); + ShortVector bv = ShortVector.fromArray(S_SPECIES, sb, 0); + VectorMask m1 = av.compare(op, bv); + m1.not().intoArray(mr, 0); + + for (int i = 0; i < S_SPECIES.length(); i++) { + verifyResults(sa[i], sb[i], mr[i], op); + } + } + + @ForceInline + public static void testCompareMaskNotInt(VectorOperators.Comparison op) { + IntVector av = IntVector.fromArray(I_SPECIES, ia, 0); + IntVector bv = IntVector.fromArray(I_SPECIES, ib, 0); + VectorMask m1 = av.compare(op, bv); + m1.not().intoArray(mr, 0); + + for (int i = 0; i < I_SPECIES.length(); i++) { + verifyResults(ia[i], ib[i], mr[i], op); + } + } + + @ForceInline + public static void testCompareMaskNotLong(VectorOperators.Comparison op) { + LongVector av = LongVector.fromArray(L_SPECIES, la, 0); + LongVector bv = LongVector.fromArray(L_SPECIES, lb, 0); + VectorMask m1 = av.compare(op, bv); + m1.not().intoArray(mr, 0); + + for (int i = 0; i < L_SPECIES.length(); i++) { + verifyResults(la[i], lb[i], mr[i], op); + } + } + + @ForceInline + public static void testCompareMaskNotFloat(VectorOperators.Comparison op, float[] a, float[] b) { + FloatVector av = FloatVector.fromArray(F_SPECIES, a, 0); + FloatVector bv = FloatVector.fromArray(F_SPECIES, b, 0); + VectorMask m1 = av.compare(op, bv); + m1.not().intoArray(mr, 0); + + for (int i = 0; i < F_SPECIES.length(); i++) { + verifyResults(a[i], b[i], mr[i], op); + } + } + + @ForceInline + public static void testCompareMaskNotDouble(VectorOperators.Comparison op, double[] a, double[] b) { + DoubleVector av = DoubleVector.fromArray(D_SPECIES, a, 0); + DoubleVector bv = DoubleVector.fromArray(D_SPECIES, b, 0); + VectorMask m1 = av.compare(op, bv); + m1.not().intoArray(mr, 0); + + for (int i = 0; i < D_SPECIES.length(); i++) { + verifyResults(a[i], b[i], mr[i], op); + } + } + + // Byte tests + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + public static void testCompareEQMaskNotByte() { + testCompareMaskNotByte(VectorOperators.EQ); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + public static void testCompareNEMaskNotByte() { + testCompareMaskNotByte(VectorOperators.NE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + public static void testCompareLTMaskNotByte() { + testCompareMaskNotByte(VectorOperators.LT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + public static void testCompareGTMaskNotByte() { + testCompareMaskNotByte(VectorOperators.GT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + public static void testCompareLEMaskNotByte() { + testCompareMaskNotByte(VectorOperators.LE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + public static void testCompareGEMaskNotByte() { + testCompareMaskNotByte(VectorOperators.GE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + public static void testCompareULTMaskNotByte() { + testCompareMaskNotByte(VectorOperators.ULT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + public static void testCompareUGTMaskNotByte() { + testCompareMaskNotByte(VectorOperators.UGT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + public static void testCompareULEMaskNotByte() { + testCompareMaskNotByte(VectorOperators.ULE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + public static void testCompareUGEMaskNotByte() { + testCompareMaskNotByte(VectorOperators.UGE); + } + + // Short tests + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + public static void testCompareEQMaskNotShort() { + testCompareMaskNotShort(VectorOperators.EQ); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + public static void testCompareNEMaskNotShort() { + testCompareMaskNotShort(VectorOperators.NE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + public static void testCompareLTMaskNotShort() { + testCompareMaskNotShort(VectorOperators.LT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + public static void testCompareGTMaskNotShort() { + testCompareMaskNotShort(VectorOperators.GT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + public static void testCompareLEMaskNotShort() { + testCompareMaskNotShort(VectorOperators.LE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + public static void testCompareGEMaskNotShort() { + testCompareMaskNotShort(VectorOperators.GE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + public static void testCompareULTMaskNotShort() { + testCompareMaskNotShort(VectorOperators.ULT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + public static void testCompareUGTMaskNotShort() { + testCompareMaskNotShort(VectorOperators.UGT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + public static void testCompareULEMaskNotShort() { + testCompareMaskNotShort(VectorOperators.ULE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + public static void testCompareUGEMaskNotShort() { + testCompareMaskNotShort(VectorOperators.UGE); + } + + // Int tests + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareEQMaskNotInt() { + testCompareMaskNotInt(VectorOperators.EQ); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareNEMaskNotInt() { + testCompareMaskNotInt(VectorOperators.NE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareLTMaskNotInt() { + testCompareMaskNotInt(VectorOperators.LT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareGTMaskNotInt() { + testCompareMaskNotInt(VectorOperators.GT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareLEMaskNotInt() { + testCompareMaskNotInt(VectorOperators.LE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareGEMaskNotInt() { + testCompareMaskNotInt(VectorOperators.GE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareULTMaskNotInt() { + testCompareMaskNotInt(VectorOperators.ULT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareUGTMaskNotInt() { + testCompareMaskNotInt(VectorOperators.UGT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareULEMaskNotInt() { + testCompareMaskNotInt(VectorOperators.ULE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareUGEMaskNotInt() { + testCompareMaskNotInt(VectorOperators.UGE); + } + + // Long tests + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareEQMaskNotLong() { + testCompareMaskNotLong(VectorOperators.EQ); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareNEMaskNotLong() { + testCompareMaskNotLong(VectorOperators.NE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareLTMaskNotLong() { + testCompareMaskNotLong(VectorOperators.LT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareGTMaskNotLong() { + testCompareMaskNotLong(VectorOperators.GT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareLEMaskNotLong() { + testCompareMaskNotLong(VectorOperators.LE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareGEMaskNotLong() { + testCompareMaskNotLong(VectorOperators.GE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareULTMaskNotLong() { + testCompareMaskNotLong(VectorOperators.ULT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareUGTMaskNotLong() { + testCompareMaskNotLong(VectorOperators.UGT); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareULEMaskNotLong() { + testCompareMaskNotLong(VectorOperators.ULE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareUGEMaskNotLong() { + testCompareMaskNotLong(VectorOperators.UGE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareEQMaskNotFloat() { + testCompareMaskNotFloat(VectorOperators.EQ, fa, fb); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + public static void testCompareNEMaskNotFloat() { + testCompareMaskNotFloat(VectorOperators.NE, fa, fb); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareEQMaskNotFloatNaN() { + testCompareMaskNotFloat(VectorOperators.EQ, fa, fnan); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareNEMaskNotFloatNaN() { + testCompareMaskNotFloat(VectorOperators.NE, fa, fnan); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareEQMaskNotFloatPositiveInfinity() { + testCompareMaskNotFloat(VectorOperators.EQ, fa, fpinf); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareNEMaskNotFloatPositiveInfinity() { + testCompareMaskNotFloat(VectorOperators.NE, fa, fpinf); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareEQMaskNotFloatNegativeInfinity() { + testCompareMaskNotFloat(VectorOperators.EQ, fa, fninf); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareNEMaskNotFloatNegativeInfinity() { + testCompareMaskNotFloat(VectorOperators.NE, fa, fninf); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareEQMaskNotDouble() { + testCompareMaskNotDouble(VectorOperators.EQ, da, db); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareNEMaskNotDouble() { + testCompareMaskNotDouble(VectorOperators.NE, da, db); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareEQMaskNotDoubleNaN() { + testCompareMaskNotDouble(VectorOperators.EQ, da, dnan); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareNEMaskNotDoubleNaN() { + testCompareMaskNotDouble(VectorOperators.NE, da, dnan); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareEQMaskNotDoublePositiveInfinity() { + testCompareMaskNotDouble(VectorOperators.EQ, da, dpinf); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareNEMaskNotDoublePositiveInfinity() { + testCompareMaskNotDouble(VectorOperators.NE, da, dpinf); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareEQMaskNotDoubleNegativeInfinity() { + testCompareMaskNotDouble(VectorOperators.EQ, da, dninf); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + public static void testCompareNEMaskNotDoubleNegativeInfinity() { + testCompareMaskNotDouble(VectorOperators.NE, da, dninf); + } + + public static void main(String[] args) { + TestFramework testFramework = new TestFramework(); + testFramework.addFlags("--add-modules=jdk.incubator.vector"); + testFramework.setDefaultWarmup(10000); + testFramework.start(); + } +} diff --git a/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java b/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java new file mode 100644 index 0000000000000..293d5bfe338a1 --- /dev/null +++ b/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.openjdk.bench.jdk.incubator.vector; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.*; + +import jdk.incubator.vector.*; +import java.util.concurrent.TimeUnit; +import java.util.Random; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Thread) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork(jvmArgs = { "--add-modules=jdk.incubator.vector" }) +public class MaskCompareNotBenchmark { + private static final int ARRAYLEN = 4096; + private static Random r = new Random(); + + private static final VectorSpecies B_SPECIES = ByteVector.SPECIES_MAX; + private static final VectorSpecies S_SPECIES = ShortVector.SPECIES_MAX; + private static final VectorSpecies I_SPECIES = IntVector.SPECIES_MAX; + private static final VectorSpecies L_SPECIES = LongVector.SPECIES_MAX; + private static final VectorSpecies F_SPECIES = FloatVector.SPECIES_MAX; + private static final VectorSpecies D_SPECIES = DoubleVector.SPECIES_MAX; + + boolean[] mr; + byte[] ba; + byte[] bb; + short[] sa; + short[] sb; + int[] ia; + int[] ib; + long[] la; + long[] lb; + float[] fa; + float[] fb; + double[] da; + double[] db; + + @Setup + public void init() { + mr = new boolean[ARRAYLEN]; + ba = new byte[ARRAYLEN]; + bb = new byte[ARRAYLEN]; + sa = new short[ARRAYLEN]; + sb = new short[ARRAYLEN]; + ia = new int[ARRAYLEN]; + ib = new int[ARRAYLEN]; + la = new long[ARRAYLEN]; + lb = new long[ARRAYLEN]; + fa = new float[ARRAYLEN]; + fb = new float[ARRAYLEN]; + da = new double[ARRAYLEN]; + db = new double[ARRAYLEN]; + + for (int i = 0; i < ARRAYLEN; i++) { + mr[i] = r.nextBoolean(); + ba[i] = (byte) r.nextInt(); + bb[i] = (byte) r.nextInt(); + sa[i] = (short) r.nextInt(); + sb[i] = (short) r.nextInt(); + ia[i] = r.nextInt(); + ib[i] = r.nextInt(); + la[i] = r.nextLong(); + lb[i] = r.nextLong(); + fa[i] = r.nextFloat(); + fb[i] = r.nextFloat(); + da[i] = r.nextDouble(); + db[i] = r.nextDouble(); + } + } + + @CompilerControl(CompilerControl.Mode.INLINE) + private void testCompareMaskNotByte(VectorOperators.Comparison op) { + ByteVector bv = ByteVector.fromArray(B_SPECIES, bb, 0); + for (int j = 0; j < ARRAYLEN; j += B_SPECIES.length()) { + ByteVector av = ByteVector.fromArray(B_SPECIES, ba, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } + + @CompilerControl(CompilerControl.Mode.INLINE) + private void testCompareMaskNotShort(VectorOperators.Comparison op) { + ShortVector bv = ShortVector.fromArray(S_SPECIES, sb, 0); + for (int j = 0; j < ARRAYLEN; j += S_SPECIES.length()) { + ShortVector av = ShortVector.fromArray(S_SPECIES, sa, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } + + @CompilerControl(CompilerControl.Mode.INLINE) + private void testCompareMaskNotInt(VectorOperators.Comparison op) { + IntVector bv = IntVector.fromArray(I_SPECIES, ib, 0); + for (int j = 0; j < ARRAYLEN; j += I_SPECIES.length()) { + IntVector av = IntVector.fromArray(I_SPECIES, ia, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } + + @CompilerControl(CompilerControl.Mode.INLINE) + private void testCompareMaskNotLong(VectorOperators.Comparison op) { + LongVector bv = LongVector.fromArray(L_SPECIES, lb, 0); + for (int j = 0; j < ARRAYLEN; j += L_SPECIES.length()) { + LongVector av = LongVector.fromArray(L_SPECIES, la, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } + + @CompilerControl(CompilerControl.Mode.INLINE) + private void testCompareMaskNotFloat(VectorOperators.Comparison op) { + FloatVector bv = FloatVector.fromArray(F_SPECIES, fb, 0); + for (int j = 0; j < ARRAYLEN; j += F_SPECIES.length()) { + FloatVector av = FloatVector.fromArray(F_SPECIES, fa, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } + + @CompilerControl(CompilerControl.Mode.INLINE) + private void testCompareMaskNotDouble(VectorOperators.Comparison op) { + DoubleVector bv = DoubleVector.fromArray(D_SPECIES, db, 0); + for (int j = 0; j < ARRAYLEN; j += D_SPECIES.length()) { + DoubleVector av = DoubleVector.fromArray(D_SPECIES, da, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } + + @Benchmark + public void testCompareEQMaskNotByte() { + testCompareMaskNotByte(VectorOperators.EQ); + } + + @Benchmark + public void testCompareNEMaskNotByte() { + testCompareMaskNotByte(VectorOperators.NE); + } + + @Benchmark + public void testCompareLTMaskNotByte() { + testCompareMaskNotByte(VectorOperators.LT); + } + + @Benchmark + public void testCompareGTMaskNotByte() { + testCompareMaskNotByte(VectorOperators.GT); + } + + @Benchmark + public void testCompareLEMaskNotByte() { + testCompareMaskNotByte(VectorOperators.LE); + } + + @Benchmark + public void testCompareGEMaskNotByte() { + testCompareMaskNotByte(VectorOperators.GE); + } + + @Benchmark + public void testCompareULTMaskNotByte() { + testCompareMaskNotByte(VectorOperators.ULT); + } + + @Benchmark + public void testCompareUGTMaskNotByte() { + testCompareMaskNotByte(VectorOperators.UGT); + } + + @Benchmark + public void testCompareULEMaskNotByte() { + testCompareMaskNotByte(VectorOperators.ULE); + } + + @Benchmark + public void testCompareUGEMaskNotByte() { + testCompareMaskNotByte(VectorOperators.UGE); + } + + @Benchmark + public void testCompareEQMaskNotShort() { + testCompareMaskNotShort(VectorOperators.EQ); + } + + @Benchmark + public void testCompareNEMaskNotShort() { + testCompareMaskNotShort(VectorOperators.NE); + } + + @Benchmark + public void testCompareLTMaskNotShort() { + testCompareMaskNotShort(VectorOperators.LT); + } + + @Benchmark + public void testCompareGTMaskNotShort() { + testCompareMaskNotShort(VectorOperators.GT); + } + + @Benchmark + public void testCompareLEMaskNotShort() { + testCompareMaskNotShort(VectorOperators.LE); + } + + @Benchmark + public void testCompareGEMaskNotShort() { + testCompareMaskNotShort(VectorOperators.GE); + } + + @Benchmark + public void testCompareULTMaskNotShort() { + testCompareMaskNotShort(VectorOperators.ULT); + } + + @Benchmark + public void testCompareUGTMaskNotShort() { + testCompareMaskNotShort(VectorOperators.UGT); + } + + @Benchmark + public void testCompareULEMaskNotShort() { + testCompareMaskNotShort(VectorOperators.ULE); + } + + @Benchmark + public void testCompareUGEMaskNotShort() { + testCompareMaskNotShort(VectorOperators.UGE); + } + + @Benchmark + public void testCompareEQMaskNotInt() { + testCompareMaskNotInt(VectorOperators.EQ); + } + + @Benchmark + public void testCompareNEMaskNotInt() { + testCompareMaskNotInt(VectorOperators.NE); + } + + @Benchmark + public void testCompareLTMaskNotInt() { + testCompareMaskNotInt(VectorOperators.LT); + } + + @Benchmark + public void testCompareGTMaskNotInt() { + testCompareMaskNotInt(VectorOperators.GT); + } + + @Benchmark + public void testCompareLEMaskNotInt() { + testCompareMaskNotInt(VectorOperators.LE); + } + + @Benchmark + public void testCompareGEMaskNotInt() { + testCompareMaskNotInt(VectorOperators.GE); + } + + @Benchmark + public void testCompareULTMaskNotInt() { + testCompareMaskNotInt(VectorOperators.ULT); + } + + @Benchmark + public void testCompareUGTMaskNotInt() { + testCompareMaskNotInt(VectorOperators.UGT); + } + + @Benchmark + public void testCompareULEMaskNotInt() { + testCompareMaskNotInt(VectorOperators.ULE); + } + + @Benchmark + public void testCompareUGEMaskNotInt() { + testCompareMaskNotInt(VectorOperators.UGE); + } + + @Benchmark + public void testCompareEQMaskNotLong() { + testCompareMaskNotLong(VectorOperators.EQ); + } + + @Benchmark + public void testCompareNEMaskNotLong() { + testCompareMaskNotLong(VectorOperators.NE); + } + + @Benchmark + public void testCompareLTMaskNotLong() { + testCompareMaskNotLong(VectorOperators.LT); + } + + @Benchmark + public void testCompareGTMaskNotLong() { + testCompareMaskNotLong(VectorOperators.GT); + } + + @Benchmark + public void testCompareLEMaskNotLong() { + testCompareMaskNotLong(VectorOperators.LE); + } + + @Benchmark + public void testCompareGEMaskNotLong() { + testCompareMaskNotLong(VectorOperators.GE); + } + + @Benchmark + public void testCompareULTMaskNotLong() { + testCompareMaskNotLong(VectorOperators.ULT); + } + + @Benchmark + public void testCompareUGTMaskNotLong() { + testCompareMaskNotLong(VectorOperators.UGT); + } + + @Benchmark + public void testCompareULEMaskNotLong() { + testCompareMaskNotLong(VectorOperators.ULE); + } + + @Benchmark + public void testCompareUGEMaskNotLong() { + testCompareMaskNotLong(VectorOperators.UGE); + } + + @Benchmark + public void testCompareEQMaskNotFloat() { + testCompareMaskNotFloat(VectorOperators.EQ); + } + + @Benchmark + public void testCompareNEMaskNotFloat() { + testCompareMaskNotFloat(VectorOperators.NE); + } + + @Benchmark + public void testCompareEQMaskNotDouble() { + testCompareMaskNotDouble(VectorOperators.EQ); + } + + @Benchmark + public void testCompareNEMaskNotDouble() { + testCompareMaskNotDouble(VectorOperators.NE); + } +} From 34eae981a6a6a388af120ba6597fdc787ca42429 Mon Sep 17 00:00:00 2001 From: erfang Date: Fri, 25 Apr 2025 07:14:22 +0000 Subject: [PATCH 02/12] Addressed some review comments 1. Call VectorNode::Ideal() only once in XorVNode::Ideal. 2. Improve code comments. --- src/hotspot/share/opto/node.cpp | 11 +++++------ src/hotspot/share/opto/vectornode.cpp | 6 ++++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index 6a5959c2c2f3f..9cf3a24864df2 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -1217,12 +1217,11 @@ bool Node::has_special_unique_user() const { // See IfNode::fold_compares return true; } else if (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) { - // Condition for removing the unnecessary not() following compare(...) operation. - // The predecessor of XorV/XorVMask may be used by an useless VectorBox node, - // making it's not single used by XorV/XorVMask. The VectorBox node will be - // eliminated by the phase RemoveUseless, but the XorV/XorVMask node may not - // be added to the IGVN worklist, then the optimization will not be applied. - // Therefore, add this node into IGVN worklist to make the optimization happen. + // Condition for removing an unnecessary not() following a compare(...) + // operation. The predecessor of n (this XorV or XorVMask) may also be used + // by a useless VectorBox node which will later be eliminated by + // RemoveUseless. Return true to ensure that subgraph transformations are + // performed on n. return true; } else { return false; diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 17fa86662acf0..d50c4aea4d1f0 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2209,9 +2209,10 @@ Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { } // Transformations for predicated IRs are not supported for now. + bool with_predicated = false; if (is_predicated_vector() || in1->is_predicated_vector() || in2->is_predicated_vector()) { - return VectorNode::Ideal(phase, can_reshape); + with_predicated = true; } // For integer types: @@ -2240,7 +2241,8 @@ Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { vmcast_vt = in1->as_Vector()->vect_type(); in1 = in1->in(1); } - if (in1->Opcode() != Op_VectorMaskCmp || in1->outcnt() > 1 || + if (with_predicated || in1->Opcode() != Op_VectorMaskCmp || + in1->outcnt() > 1 || !((VectorMaskCmpNode*) in1)->predicate_can_be_inverted() || !VectorNode::is_all_ones_vector(in2)) { return VectorNode::Ideal(phase, can_reshape); From 4fbf84e39cfd13413ba188f2d198e1872131d770 Mon Sep 17 00:00:00 2001 From: erfang Date: Thu, 1 May 2025 07:28:39 +0000 Subject: [PATCH 03/12] Update the jtreg test --- .../vectorapi/VectorMaskCompareNotTest.java | 169 ++++++++++++------ 1 file changed, 112 insertions(+), 57 deletions(-) diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java index 10c729128032d..f507abb6d6d72 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java @@ -35,7 +35,6 @@ * @library /test/lib / * @summary test combining vector not operation with compare * @modules jdk.incubator.vector - * @requires ((os.arch!="x86" & os.arch!="i386" & os.arch!="amd64" & os.arch!="x86_64") | vm.cpu.features ~= ".*avx.*") * * @run driver compiler.vectorapi.VectorMaskCompareNotTest */ @@ -231,340 +230,396 @@ public static void testCompareMaskNotDouble(VectorOperators.Comparison op, doubl // Byte tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotByte() { testCompareMaskNotByte(VectorOperators.EQ); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotByte() { testCompareMaskNotByte(VectorOperators.NE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotByte() { testCompareMaskNotByte(VectorOperators.LT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotByte() { testCompareMaskNotByte(VectorOperators.GT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotByte() { testCompareMaskNotByte(VectorOperators.LE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotByte() { testCompareMaskNotByte(VectorOperators.GE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotByte() { testCompareMaskNotByte(VectorOperators.ULT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotByte() { testCompareMaskNotByte(VectorOperators.UGT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotByte() { testCompareMaskNotByte(VectorOperators.ULE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotByte() { testCompareMaskNotByte(VectorOperators.UGE); } // Short tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotShort() { testCompareMaskNotShort(VectorOperators.EQ); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotShort() { testCompareMaskNotShort(VectorOperators.NE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotShort() { testCompareMaskNotShort(VectorOperators.LT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotShort() { testCompareMaskNotShort(VectorOperators.GT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotShort() { testCompareMaskNotShort(VectorOperators.LE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotShort() { testCompareMaskNotShort(VectorOperators.GE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotShort() { testCompareMaskNotShort(VectorOperators.ULT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotShort() { testCompareMaskNotShort(VectorOperators.UGT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotShort() { testCompareMaskNotShort(VectorOperators.ULE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotShort() { testCompareMaskNotShort(VectorOperators.UGE); } // Int tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotInt() { testCompareMaskNotInt(VectorOperators.EQ); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotInt() { testCompareMaskNotInt(VectorOperators.NE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotInt() { testCompareMaskNotInt(VectorOperators.LT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotInt() { testCompareMaskNotInt(VectorOperators.GT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotInt() { testCompareMaskNotInt(VectorOperators.LE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotInt() { testCompareMaskNotInt(VectorOperators.GE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotInt() { testCompareMaskNotInt(VectorOperators.ULT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotInt() { testCompareMaskNotInt(VectorOperators.UGT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotInt() { testCompareMaskNotInt(VectorOperators.ULE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotInt() { testCompareMaskNotInt(VectorOperators.UGE); } // Long tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotLong() { testCompareMaskNotLong(VectorOperators.EQ); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotLong() { testCompareMaskNotLong(VectorOperators.NE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotLong() { testCompareMaskNotLong(VectorOperators.LT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotLong() { testCompareMaskNotLong(VectorOperators.GT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotLong() { testCompareMaskNotLong(VectorOperators.LE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotLong() { testCompareMaskNotLong(VectorOperators.GE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotLong() { testCompareMaskNotLong(VectorOperators.ULT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotLong() { testCompareMaskNotLong(VectorOperators.UGT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotLong() { testCompareMaskNotLong(VectorOperators.ULE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotLong() { testCompareMaskNotLong(VectorOperators.UGE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloat() { testCompareMaskNotFloat(VectorOperators.EQ, fa, fb); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloat() { testCompareMaskNotFloat(VectorOperators.NE, fa, fb); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloatNaN() { testCompareMaskNotFloat(VectorOperators.EQ, fa, fnan); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloatNaN() { testCompareMaskNotFloat(VectorOperators.NE, fa, fnan); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloatPositiveInfinity() { testCompareMaskNotFloat(VectorOperators.EQ, fa, fpinf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloatPositiveInfinity() { testCompareMaskNotFloat(VectorOperators.NE, fa, fpinf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloatNegativeInfinity() { testCompareMaskNotFloat(VectorOperators.EQ, fa, fninf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloatNegativeInfinity() { testCompareMaskNotFloat(VectorOperators.NE, fa, fninf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotDouble() { testCompareMaskNotDouble(VectorOperators.EQ, da, db); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotDouble() { testCompareMaskNotDouble(VectorOperators.NE, da, db); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotDoubleNaN() { testCompareMaskNotDouble(VectorOperators.EQ, da, dnan); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotDoubleNaN() { testCompareMaskNotDouble(VectorOperators.NE, da, dnan); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotDoublePositiveInfinity() { testCompareMaskNotDouble(VectorOperators.EQ, da, dpinf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotDoublePositiveInfinity() { testCompareMaskNotDouble(VectorOperators.NE, da, dpinf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotDoubleNegativeInfinity() { testCompareMaskNotDouble(VectorOperators.EQ, da, dninf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotDoubleNegativeInfinity() { testCompareMaskNotDouble(VectorOperators.NE, da, dninf); } From 001fac0f2a94d56bd3e670762ae5e5d5acc17327 Mon Sep 17 00:00:00 2001 From: erfang Date: Wed, 7 May 2025 02:00:16 +0000 Subject: [PATCH 04/12] Refactor code Add a new function XorVNode::Ideal_XorV_VectorMaskCmp to do this optimization, making the code more modular. --- src/hotspot/share/opto/vectornode.cpp | 76 +++++++++++++++------------ src/hotspot/share/opto/vectornode.hpp | 1 + 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index a3386c036bd79..8bc8f5af0b9db 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2191,45 +2191,31 @@ bool VectorMaskCmpNode::predicate_can_be_inverted() { } } -Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { +// This function transforms the following patterns: +// +// For integer types: +// (XorV (VectorMaskCmp src1 src2 cond) (Replicate -1)) +// => (VectorMaskCmp src1 src2 ncond) +// (XorVMask (VectorMaskCmp src1 src2 cond) (MaskAll m1)) +// => (VectorMaskCmp src1 src2 ncond) +// cond can be eq, ne, le, ge, lt, gt, ule, uge, ult and ugt. +// ncond is the negative comparison of cond. +// +// For float and double types: +// (XorV (VectorMaskCast (VectorMaskCmp src1 src2 cond)) (Replicate -1)) +// => (VectorMaskCast (VectorMaskCmp src1 src2 ncond)) +// (XorVMask (VectorMaskCast (VectorMaskCmp src1 src2 cond)) (MaskAll m1)) +// => (VectorMaskCast (VectorMaskCmp src1 src2 ncond)) +// cond can be eq or ne. +Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { Node* in1 = in(1); Node* in2 = in(2); - // (XorV src src) => (Replicate zero) - // (XorVMask src src) => (MaskAll zero) - // - // The transformation is only applied to the un-predicated - // operation, since the VectorAPI masked operation requires - // the unmasked lanes to save the same values in the first - // operand. - if (!is_predicated_vector() && (in1 == in2)) { - BasicType bt = vect_type()->element_basic_type(); - Node* zero = phase->transform(phase->zerocon(bt)); - return VectorNode::scalar2vector(zero, length(), bt, - bottom_type()->isa_vectmask() != nullptr); - } - // Transformations for predicated IRs are not supported for now. - bool with_predicated = false; if (is_predicated_vector() || in1->is_predicated_vector() || in2->is_predicated_vector()) { - with_predicated = true; + return nullptr; } - // For integer types: - // (XorV (VectorMaskCmp src1 src2 cond) (Replicate -1)) - // => (VectorMaskCmp src1 src2 ncond) - // (XorVMask (VectorMaskCmp src1 src2 cond) (MaskAll m1)) - // => (VectorMaskCmp src1 src2 ncond) - // cond can be eq, ne, le, ge, lt, gt, ule, uge, ult and ugt, ncond is the - // negative comparison of cond. - // - // For float and double types: - // (XorV (VectorMaskCast (VectorMaskCmp src1 src2 cond)) (Replicate -1)) - // => (VectorMaskCast (VectorMaskCmp src1 src2 ncond)) - // (XorVMask (VectorMaskCast (VectorMaskCmp src1 src2 cond)) (MaskAll m1)) - // => (VectorMaskCast (VectorMaskCmp src1 src2 ncond)) - // cond can be eq or ne. - // // XorV/XorVMask is commutative, swap VectorMaskCmp/Op_VectorMaskCast to in1. if (in1->Opcode() != Op_VectorMaskCmp && in1->Opcode() != Op_VectorMaskCast) { swap(in1, in2); @@ -2241,11 +2227,10 @@ Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { vmcast_vt = in1->as_Vector()->vect_type(); in1 = in1->in(1); } - if (with_predicated || in1->Opcode() != Op_VectorMaskCmp || - in1->outcnt() > 1 || + if (in1->Opcode() != Op_VectorMaskCmp || in1->outcnt() > 1 || !((VectorMaskCmpNode*) in1)->predicate_can_be_inverted() || !VectorNode::is_all_ones_vector(in2)) { - return VectorNode::Ideal(phase, can_reshape); + return nullptr; } // This is the same with BoolTest::negate(), but we can't call it with a @@ -2267,6 +2252,27 @@ Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { return vmcmp; } +Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { + // (XorV src src) => (Replicate zero) + // (XorVMask src src) => (MaskAll zero) + // + // The transformation is only applied to the un-predicated + // operation, since the VectorAPI masked operation requires + // the unmasked lanes to save the same values in the first + // operand. + if (!is_predicated_vector() && (in(1) == in(2))) { + BasicType bt = vect_type()->element_basic_type(); + Node* zero = phase->transform(phase->zerocon(bt)); + return VectorNode::scalar2vector(zero, length(), bt, bottom_type()->isa_vectmask() != nullptr); + } + + Node* res = Ideal_XorV_VectorMaskCmp(phase, can_reshape); + if (res == nullptr) { + res = VectorNode::Ideal(phase, can_reshape); + } + return res; +} + Node* VectorBlendNode::Identity(PhaseGVN* phase) { // (VectorBlend X X MASK) => X if (in(1) == in(2)) { diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index a2c0088fb2d49..05ad6cb4820db 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1012,6 +1012,7 @@ class XorVNode : public VectorNode { XorVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {} virtual int Opcode() const; virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); + Node* Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape); }; //------------------------------XorReductionVNode-------------------------------------- From f2f71e3467721b88f72efab449efd62a5ecd71b0 Mon Sep 17 00:00:00 2001 From: erfang Date: Wed, 14 May 2025 02:35:09 +0000 Subject: [PATCH 05/12] Refactor the JTReg tests for compare.xor(maskAll) Also made a bit change to support pattern `VectorMask.fromLong()`. --- src/hotspot/share/opto/vectornode.cpp | 6 +- .../vectorapi/VectorMaskCompareNotTest.java | 196 ++++++++++++------ .../vector/MaskCompareNotBenchmark.java | 3 +- 3 files changed, 135 insertions(+), 70 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 8bc8f5af0b9db..994f0aa55b641 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2217,7 +2217,8 @@ Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { } // XorV/XorVMask is commutative, swap VectorMaskCmp/Op_VectorMaskCast to in1. - if (in1->Opcode() != Op_VectorMaskCmp && in1->Opcode() != Op_VectorMaskCast) { + if (in2->Opcode() == Op_VectorMaskCmp || + (in2->Opcode() == Op_VectorMaskCast && in2->in(1)->Opcode() == Op_VectorMaskCmp)) { swap(in1, in2); } @@ -2227,6 +2228,9 @@ Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { vmcast_vt = in1->as_Vector()->vect_type(); in1 = in1->in(1); } + if (in2->Opcode() == Op_VectorMaskCast) { + in2 = in2->in(1); + } if (in1->Opcode() != Op_VectorMaskCmp || in1->outcnt() > 1 || !((VectorMaskCmpNode*) in1)->predicate_can_be_inverted() || !VectorNode::is_all_ones_vector(in2)) { diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java index f507abb6d6d72..34bc943136c00 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java @@ -156,12 +156,16 @@ public static > void verifyResults(T a, T b, bo } } + interface VectorMaskOperator { + public VectorMask apply(VectorMask m1); + } + @ForceInline - public static void testCompareMaskNotByte(VectorOperators.Comparison op) { + public static void testCompareMaskNotByte(VectorOperators.Comparison op, VectorMaskOperator func) { ByteVector av = ByteVector.fromArray(B_SPECIES, ba, 0); ByteVector bv = ByteVector.fromArray(B_SPECIES, bb, 0); VectorMask m1 = av.compare(op, bv); - m1.not().intoArray(mr, 0); + func.apply(m1).intoArray(mr, 0); for (int i = 0; i < B_SPECIES.length(); i++) { verifyResults(ba[i], bb[i], mr[i], op); @@ -169,11 +173,11 @@ public static void testCompareMaskNotByte(VectorOperators.Comparison op) { } @ForceInline - public static void testCompareMaskNotShort(VectorOperators.Comparison op) { + public static void testCompareMaskNotShort(VectorOperators.Comparison op, VectorMaskOperator func) { ShortVector av = ShortVector.fromArray(S_SPECIES, sa, 0); ShortVector bv = ShortVector.fromArray(S_SPECIES, sb, 0); VectorMask m1 = av.compare(op, bv); - m1.not().intoArray(mr, 0); + func.apply(m1).intoArray(mr, 0); for (int i = 0; i < S_SPECIES.length(); i++) { verifyResults(sa[i], sb[i], mr[i], op); @@ -181,11 +185,11 @@ public static void testCompareMaskNotShort(VectorOperators.Comparison op) { } @ForceInline - public static void testCompareMaskNotInt(VectorOperators.Comparison op) { + public static void testCompareMaskNotInt(VectorOperators.Comparison op, VectorMaskOperator func) { IntVector av = IntVector.fromArray(I_SPECIES, ia, 0); IntVector bv = IntVector.fromArray(I_SPECIES, ib, 0); VectorMask m1 = av.compare(op, bv); - m1.not().intoArray(mr, 0); + func.apply(m1).intoArray(mr, 0); for (int i = 0; i < I_SPECIES.length(); i++) { verifyResults(ia[i], ib[i], mr[i], op); @@ -193,11 +197,11 @@ public static void testCompareMaskNotInt(VectorOperators.Comparison op) { } @ForceInline - public static void testCompareMaskNotLong(VectorOperators.Comparison op) { + public static void testCompareMaskNotLong(VectorOperators.Comparison op, VectorMaskOperator func) { LongVector av = LongVector.fromArray(L_SPECIES, la, 0); LongVector bv = LongVector.fromArray(L_SPECIES, lb, 0); VectorMask m1 = av.compare(op, bv); - m1.not().intoArray(mr, 0); + func.apply(m1).intoArray(mr, 0); for (int i = 0; i < L_SPECIES.length(); i++) { verifyResults(la[i], lb[i], mr[i], op); @@ -205,11 +209,11 @@ public static void testCompareMaskNotLong(VectorOperators.Comparison op) { } @ForceInline - public static void testCompareMaskNotFloat(VectorOperators.Comparison op, float[] a, float[] b) { + public static void testCompareMaskNotFloat(VectorOperators.Comparison op, float[] a, float[] b, VectorMaskOperator func) { FloatVector av = FloatVector.fromArray(F_SPECIES, a, 0); FloatVector bv = FloatVector.fromArray(F_SPECIES, b, 0); VectorMask m1 = av.compare(op, bv); - m1.not().intoArray(mr, 0); + func.apply(m1).intoArray(mr, 0); for (int i = 0; i < F_SPECIES.length(); i++) { verifyResults(a[i], b[i], mr[i], op); @@ -217,11 +221,11 @@ public static void testCompareMaskNotFloat(VectorOperators.Comparison op, float[ } @ForceInline - public static void testCompareMaskNotDouble(VectorOperators.Comparison op, double[] a, double[] b) { + public static void testCompareMaskNotDouble(VectorOperators.Comparison op, double[] a, double[] b, VectorMaskOperator func) { DoubleVector av = DoubleVector.fromArray(D_SPECIES, a, 0); DoubleVector bv = DoubleVector.fromArray(D_SPECIES, b, 0); VectorMask m1 = av.compare(op, bv); - m1.not().intoArray(mr, 0); + func.apply(m1).intoArray(mr, 0); for (int i = 0; i < D_SPECIES.length(); i++) { verifyResults(a[i], b[i], mr[i], op); @@ -233,70 +237,80 @@ public static void testCompareMaskNotDouble(VectorOperators.Comparison op, doubl @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotByte() { - testCompareMaskNotByte(VectorOperators.EQ); + testCompareMaskNotByte(VectorOperators.EQ, m -> m.not()); + testCompareMaskNotByte(VectorOperators.EQ, m -> m.xor(B_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.NE); + testCompareMaskNotByte(VectorOperators.NE, m -> m.not()); + testCompareMaskNotByte(VectorOperators.NE, m -> m.xor(B_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.LT); + testCompareMaskNotByte(VectorOperators.LT, m -> m.not()); + testCompareMaskNotByte(VectorOperators.LT, m -> m.xor(B_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.GT); + testCompareMaskNotByte(VectorOperators.GT, m -> m.not()); + testCompareMaskNotByte(VectorOperators.GT, m -> m.xor(B_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.LE); + testCompareMaskNotByte(VectorOperators.LE, m -> m.not()); + testCompareMaskNotByte(VectorOperators.LE, m -> m.xor(B_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.GE); + testCompareMaskNotByte(VectorOperators.GE, m -> m.not()); + testCompareMaskNotByte(VectorOperators.GE, m -> m.xor(B_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.ULT); + testCompareMaskNotByte(VectorOperators.ULT, m -> m.not()); + testCompareMaskNotByte(VectorOperators.ULT, m -> m.xor(B_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.UGT); + testCompareMaskNotByte(VectorOperators.UGT, m -> m.not()); + testCompareMaskNotByte(VectorOperators.UGT, m -> m.xor(B_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.ULE); + testCompareMaskNotByte(VectorOperators.ULE, m -> m.not()); + testCompareMaskNotByte(VectorOperators.ULE, m -> m.xor(B_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.UGE); + testCompareMaskNotByte(VectorOperators.UGE, m -> m.not()); + testCompareMaskNotByte(VectorOperators.UGE, m -> m.xor(B_SPECIES.maskAll(true))); } // Short tests @@ -304,70 +318,80 @@ public static void testCompareUGEMaskNotByte() { @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotShort() { - testCompareMaskNotShort(VectorOperators.EQ); + testCompareMaskNotShort(VectorOperators.EQ, m -> m.not()); + testCompareMaskNotShort(VectorOperators.EQ, m -> m.xor(S_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.NE); + testCompareMaskNotShort(VectorOperators.NE, m -> m.not()); + testCompareMaskNotShort(VectorOperators.NE, m -> m.xor(S_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.LT); + testCompareMaskNotShort(VectorOperators.LT, m -> m.not()); + testCompareMaskNotShort(VectorOperators.LT, m -> m.xor(S_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.GT); + testCompareMaskNotShort(VectorOperators.GT, m -> m.not()); + testCompareMaskNotShort(VectorOperators.GT, m -> m.xor(S_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.LE); + testCompareMaskNotShort(VectorOperators.LE, m -> m.not()); + testCompareMaskNotShort(VectorOperators.LE, m -> m.xor(S_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.GE); + testCompareMaskNotShort(VectorOperators.GE, m -> m.not()); + testCompareMaskNotShort(VectorOperators.GE, m -> m.xor(S_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.ULT); + testCompareMaskNotShort(VectorOperators.ULT, m -> m.not()); + testCompareMaskNotShort(VectorOperators.ULT, m -> m.xor(S_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.UGT); + testCompareMaskNotShort(VectorOperators.UGT, m -> m.not()); + testCompareMaskNotShort(VectorOperators.UGT, m -> m.xor(S_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.ULE); + testCompareMaskNotShort(VectorOperators.ULE, m -> m.not()); + testCompareMaskNotShort(VectorOperators.ULE, m -> m.xor(S_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.UGE); + testCompareMaskNotShort(VectorOperators.UGE, m -> m.not()); + testCompareMaskNotShort(VectorOperators.UGE, m -> m.xor(S_SPECIES.maskAll(true))); } // Int tests @@ -375,70 +399,80 @@ public static void testCompareUGEMaskNotShort() { @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotInt() { - testCompareMaskNotInt(VectorOperators.EQ); + testCompareMaskNotInt(VectorOperators.EQ, m -> m.not()); + testCompareMaskNotInt(VectorOperators.EQ, m -> m.xor(I_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.NE); + testCompareMaskNotInt(VectorOperators.NE, m -> m.not()); + testCompareMaskNotInt(VectorOperators.NE, m -> m.xor(I_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.LT); + testCompareMaskNotInt(VectorOperators.LT, m -> m.not()); + testCompareMaskNotInt(VectorOperators.LT, m -> m.xor(I_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.GT); + testCompareMaskNotInt(VectorOperators.GT, m -> m.not()); + testCompareMaskNotInt(VectorOperators.GT, m -> m.xor(I_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.LE); + testCompareMaskNotInt(VectorOperators.LE, m -> m.not()); + testCompareMaskNotInt(VectorOperators.LE, m -> m.xor(I_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.GE); + testCompareMaskNotInt(VectorOperators.GE, m -> m.not()); + testCompareMaskNotInt(VectorOperators.GE, m -> m.xor(I_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.ULT); + testCompareMaskNotInt(VectorOperators.ULT, m -> m.not()); + testCompareMaskNotInt(VectorOperators.ULT, m -> m.xor(I_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.UGT); + testCompareMaskNotInt(VectorOperators.UGT, m -> m.not()); + testCompareMaskNotInt(VectorOperators.UGT, m -> m.xor(I_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.ULE); + testCompareMaskNotInt(VectorOperators.ULE, m -> m.not()); + testCompareMaskNotInt(VectorOperators.ULE, m -> m.xor(I_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.UGE); + testCompareMaskNotInt(VectorOperators.UGE, m -> m.not()); + testCompareMaskNotInt(VectorOperators.UGE, m -> m.xor(I_SPECIES.maskAll(true))); } // Long tests @@ -446,182 +480,208 @@ public static void testCompareUGEMaskNotInt() { @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotLong() { - testCompareMaskNotLong(VectorOperators.EQ); + testCompareMaskNotLong(VectorOperators.EQ, m -> m.not()); + testCompareMaskNotLong(VectorOperators.EQ, m -> m.xor(L_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.NE); + testCompareMaskNotLong(VectorOperators.NE, m -> m.not()); + testCompareMaskNotLong(VectorOperators.NE, m -> m.xor(L_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.LT); + testCompareMaskNotLong(VectorOperators.LT, m -> m.not()); + testCompareMaskNotLong(VectorOperators.LT, m -> m.xor(L_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.GT); + testCompareMaskNotLong(VectorOperators.GT, m -> m.not()); + testCompareMaskNotLong(VectorOperators.GT, m -> m.xor(L_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.LE); + testCompareMaskNotLong(VectorOperators.LE, m -> m.not()); + testCompareMaskNotLong(VectorOperators.LE, m -> m.xor(L_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.GE); + testCompareMaskNotLong(VectorOperators.GE, m -> m.not()); + testCompareMaskNotLong(VectorOperators.GE, m -> m.xor(L_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.ULT); + testCompareMaskNotLong(VectorOperators.ULT, m -> m.not()); + testCompareMaskNotLong(VectorOperators.ULT, m -> m.xor(L_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.UGT); + testCompareMaskNotLong(VectorOperators.UGT, m -> m.not()); + testCompareMaskNotLong(VectorOperators.UGT, m -> m.xor(L_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.ULE); + testCompareMaskNotLong(VectorOperators.ULE, m -> m.not()); + testCompareMaskNotLong(VectorOperators.ULE, m -> m.xor(L_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.UGE); + testCompareMaskNotLong(VectorOperators.UGE, m -> m.not()); + testCompareMaskNotLong(VectorOperators.UGE, m -> m.xor(L_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloat() { - testCompareMaskNotFloat(VectorOperators.EQ, fa, fb); + testCompareMaskNotFloat(VectorOperators.EQ, fa, fb, m -> m.not()); + testCompareMaskNotFloat(VectorOperators.EQ, fa, fb, m -> m.xor(F_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloat() { - testCompareMaskNotFloat(VectorOperators.NE, fa, fb); + testCompareMaskNotFloat(VectorOperators.NE, fa, fb, m -> m.not()); + testCompareMaskNotFloat(VectorOperators.NE, fa, fb, m -> m.xor(F_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloatNaN() { - testCompareMaskNotFloat(VectorOperators.EQ, fa, fnan); + testCompareMaskNotFloat(VectorOperators.EQ, fa, fnan, m -> m.not()); + testCompareMaskNotFloat(VectorOperators.EQ, fa, fnan, m -> m.xor(F_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloatNaN() { - testCompareMaskNotFloat(VectorOperators.NE, fa, fnan); + testCompareMaskNotFloat(VectorOperators.NE, fa, fnan, m -> m.not()); + testCompareMaskNotFloat(VectorOperators.NE, fa, fnan, m -> m.xor(F_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloatPositiveInfinity() { - testCompareMaskNotFloat(VectorOperators.EQ, fa, fpinf); + testCompareMaskNotFloat(VectorOperators.EQ, fa, fpinf, m -> m.not()); + testCompareMaskNotFloat(VectorOperators.EQ, fa, fpinf, m -> m.xor(F_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloatPositiveInfinity() { - testCompareMaskNotFloat(VectorOperators.NE, fa, fpinf); + testCompareMaskNotFloat(VectorOperators.NE, fa, fpinf, m -> m.not()); + testCompareMaskNotFloat(VectorOperators.NE, fa, fpinf, m -> m.xor(F_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloatNegativeInfinity() { - testCompareMaskNotFloat(VectorOperators.EQ, fa, fninf); + testCompareMaskNotFloat(VectorOperators.EQ, fa, fninf, m -> m.not()); + testCompareMaskNotFloat(VectorOperators.EQ, fa, fninf, m -> m.xor(F_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloatNegativeInfinity() { - testCompareMaskNotFloat(VectorOperators.NE, fa, fninf); + testCompareMaskNotFloat(VectorOperators.NE, fa, fninf, m -> m.not()); + testCompareMaskNotFloat(VectorOperators.NE, fa, fninf, m -> m.xor(F_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotDouble() { - testCompareMaskNotDouble(VectorOperators.EQ, da, db); + testCompareMaskNotDouble(VectorOperators.EQ, da, db, m -> m.not()); + testCompareMaskNotDouble(VectorOperators.EQ, da, db, m -> m.xor(D_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotDouble() { - testCompareMaskNotDouble(VectorOperators.NE, da, db); + testCompareMaskNotDouble(VectorOperators.NE, da, db, m -> m.not()); + testCompareMaskNotDouble(VectorOperators.NE, da, db, m -> m.xor(D_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotDoubleNaN() { - testCompareMaskNotDouble(VectorOperators.EQ, da, dnan); + testCompareMaskNotDouble(VectorOperators.EQ, da, dnan, m -> m.not()); + testCompareMaskNotDouble(VectorOperators.EQ, da, dnan, m -> m.xor(D_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotDoubleNaN() { - testCompareMaskNotDouble(VectorOperators.NE, da, dnan); + testCompareMaskNotDouble(VectorOperators.NE, da, dnan, m -> m.not()); + testCompareMaskNotDouble(VectorOperators.NE, da, dnan, m -> m.xor(D_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotDoublePositiveInfinity() { - testCompareMaskNotDouble(VectorOperators.EQ, da, dpinf); + testCompareMaskNotDouble(VectorOperators.EQ, da, dpinf, m -> m.not()); + testCompareMaskNotDouble(VectorOperators.EQ, da, dpinf, m -> m.xor(D_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotDoublePositiveInfinity() { - testCompareMaskNotDouble(VectorOperators.NE, da, dpinf); + testCompareMaskNotDouble(VectorOperators.NE, da, dpinf, m -> m.not()); + testCompareMaskNotDouble(VectorOperators.NE, da, dpinf, m -> m.xor(D_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotDoubleNegativeInfinity() { - testCompareMaskNotDouble(VectorOperators.EQ, da, dninf); + testCompareMaskNotDouble(VectorOperators.EQ, da, dninf, m -> m.not()); + testCompareMaskNotDouble(VectorOperators.EQ, da, dninf, m -> m.xor(D_SPECIES.maskAll(true))); } @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotDoubleNegativeInfinity() { - testCompareMaskNotDouble(VectorOperators.NE, da, dninf); + testCompareMaskNotDouble(VectorOperators.NE, da, dninf, m -> m.not()); + testCompareMaskNotDouble(VectorOperators.NE, da, dninf, m -> m.xor(D_SPECIES.maskAll(true))); } public static void main(String[] args) { diff --git a/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java b/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java index 293d5bfe338a1..9e60fc81cfcd2 100644 --- a/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java +++ b/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java @@ -37,7 +37,8 @@ @Measurement(iterations = 5, time = 1) @Fork(jvmArgs = { "--add-modules=jdk.incubator.vector" }) public class MaskCompareNotBenchmark { - private static final int ARRAYLEN = 4096; + @Param({"4096"}) + private int ARRAYLEN; private static Random r = new Random(); private static final VectorSpecies B_SPECIES = ByteVector.SPECIES_MAX; From ebbcc405018f0261c3a7f860ea6f4f6b54bbd643 Mon Sep 17 00:00:00 2001 From: erfang Date: Thu, 5 Jun 2025 09:04:25 +0000 Subject: [PATCH 06/12] Addressed some review comments --- src/hotspot/share/opto/vectornode.cpp | 49 +++-- src/hotspot/share/opto/vectornode.hpp | 5 +- .../compiler/lib/ir_framework/IRNode.java | 10 + .../checkattribute/parsing/RawIRNode.java | 4 +- .../vectorapi/VectorMaskCompareNotTest.java | 178 +++++++++--------- 5 files changed, 128 insertions(+), 118 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 1dbb7a5dd3b33..51bd9f67a50c8 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2163,9 +2163,9 @@ Node* OrVNode::Identity(PhaseGVN* phase) { return redundant_logical_identity(this); } -// Returns whether (XorV (VectorMaskCmp) -1) can be optimized by performing the -// inverse of a comparison operation. -bool VectorMaskCmpNode::predicate_can_be_inverted() { +// Returns whether (XorV (VectorMaskCmp) -1) can be optimized by negating the +// comparison operation. +bool VectorMaskCmpNode::predicate_can_be_negated() { switch (_predicate) { case BoolTest::eq: case BoolTest::ne: @@ -2209,50 +2209,45 @@ bool VectorMaskCmpNode::predicate_can_be_inverted() { Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { Node* in1 = in(1); Node* in2 = in(2); - // Transformations for predicated IRs are not supported for now. - if (is_predicated_vector() || in1->is_predicated_vector() || + // Transformations for predicated vectors are not supported for now. + if (is_predicated_vector() || + in1->is_predicated_vector() || in2->is_predicated_vector()) { return nullptr; } - // XorV/XorVMask is commutative, swap VectorMaskCmp/Op_VectorMaskCast to in1. + // XorV/XorVMask is commutative, swap VectorMaskCmp/VectorMaskCast to in1. if (in2->Opcode() == Op_VectorMaskCmp || (in2->Opcode() == Op_VectorMaskCast && in2->in(1)->Opcode() == Op_VectorMaskCmp)) { swap(in1, in2); } - const TypeVect* vmcast_vt = nullptr; - if (in1->Opcode() == Op_VectorMaskCast && in1->outcnt() == 1 && - in1->in(1)->Opcode() == Op_VectorMaskCmp) { - vmcast_vt = in1->as_Vector()->vect_type(); + const TypeVect* vector_mask_cast_vt = nullptr; + // in1 should be single used, otherwise the optimization may be unprofitable. + if (in1->Opcode() == Op_VectorMaskCast && in1->outcnt() == 1 && in1->in(1)->Opcode() == Op_VectorMaskCmp) { + vector_mask_cast_vt = in1->as_Vector()->vect_type(); in1 = in1->in(1); } - if (in2->Opcode() == Op_VectorMaskCast) { - in2 = in2->in(1); - } - if (in1->Opcode() != Op_VectorMaskCmp || in1->outcnt() > 1 || - !((VectorMaskCmpNode*) in1)->predicate_can_be_inverted() || + + if (in1->Opcode() != Op_VectorMaskCmp || + in1->outcnt() > 1 || + !((VectorMaskCmpNode*) in1)->predicate_can_be_negated() || !VectorNode::is_all_ones_vector(in2)) { return nullptr; } - // This is the same with BoolTest::negate(), but we can't call it with a - // BoolTest object because the comparison may be unsigned comparison, but - // BoolTest doesn't support unsigned comparisons. - BoolTest::mask neg_cond = - (BoolTest::mask) (((VectorMaskCmpNode*) in1)->get_predicate() ^ 4); - + BoolTest::mask neg_cond = (BoolTest::mask) (((VectorMaskCmpNode*) in1)->get_negative_predicate()); ConINode* predicate_node = phase->intcon(neg_cond); const TypeVect* vt = in1->as_Vector()->vect_type(); - Node* vmcmp = new VectorMaskCmpNode(neg_cond, in1->in(1), in1->in(2), + Node* res = new VectorMaskCmpNode(neg_cond, in1->in(1), in1->in(2), predicate_node, vt); - if (vmcast_vt != nullptr) { - // We optimized out an VectorMaskCast, and in order to ensure type + if (vector_mask_cast_vt != nullptr) { + // We optimized out a VectorMaskCast, and in order to ensure type // correctness, we need to regenerate one. VectorMaskCast will be encoded as - // empty for types with the same size. - vmcmp = new VectorMaskCastNode(phase->transform(vmcmp), vmcast_vt); + // a no-op (identity function) for types with the same size. + res = new VectorMaskCastNode(phase->transform(res), vector_mask_cast_vt); } - return vmcmp; + return res; } Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 05ad6cb4820db..015029331fc74 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1685,8 +1685,11 @@ class VectorMaskCmpNode : public VectorNode { virtual bool cmp( const Node &n ) const { return VectorNode::cmp(n) && _predicate == ((VectorMaskCmpNode&)n)._predicate; } - bool predicate_can_be_inverted(); + bool predicate_can_be_negated(); BoolTest::mask get_predicate() { return _predicate; } + // This is the same with BoolTest::negate(), but BoolTest doesn't support + // unsigned comparison. However _predicate can be unsigned comparison. + BoolTest::mask get_negative_predicate() { return BoolTest::mask(_predicate ^ 4); } #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; #endif // !PRODUCT diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index 96b62a2b12a72..5d839a626eeea 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -2226,6 +2226,16 @@ public class IRNode { vectorNode(VECTOR_BLEND_D, "VectorBlend", TYPE_DOUBLE); } + public static final String VECTOR_MASK_CMP_B = VECTOR_PREFIX + "VECTOR_MASK_CMP_B" + POSTFIX; + static { + vectorNode(VECTOR_MASK_CMP_B, "VectorMaskCmp", TYPE_BYTE); + } + + public static final String VECTOR_MASK_CMP_S = VECTOR_PREFIX + "VECTOR_MASK_CMP_S" + POSTFIX; + static { + vectorNode(VECTOR_MASK_CMP_S, "VectorMaskCmp", TYPE_SHORT); + } + public static final String VECTOR_MASK_CMP_I = VECTOR_PREFIX + "VECTOR_MASK_CMP_I" + POSTFIX; static { vectorNode(VECTOR_MASK_CMP_I, "VectorMaskCmp", TYPE_INT); diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/driver/irmatching/irrule/checkattribute/parsing/RawIRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/driver/irmatching/irrule/checkattribute/parsing/RawIRNode.java index bf3021a6868f1..ae06f2edd470a 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/driver/irmatching/irrule/checkattribute/parsing/RawIRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/driver/irmatching/irrule/checkattribute/parsing/RawIRNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -133,6 +133,6 @@ private String regexForVectorIRNode(String nodeRegex, VMInfo vmInfo, Comparison. } } String sizeRegex = IRNode.parseVectorNodeSize(size, type, vmInfo); - return nodeRegex.replaceAll(IRNode.IS_REPLACED, "vector[A-Za-z]<" + type + "," + sizeRegex + ">"); + return nodeRegex.replaceAll(IRNode.IS_REPLACED, "vector([A-Za-z]|mask)<" + type + "," + sizeRegex + ">"); } } diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java index 34bc943136c00..3d9d710721e6a 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java @@ -41,12 +41,12 @@ public class VectorMaskCompareNotTest { private static int LENGTH = 128; - private static final VectorSpecies B_SPECIES = ByteVector.SPECIES_MAX; - private static final VectorSpecies S_SPECIES = ShortVector.SPECIES_MAX; - private static final VectorSpecies I_SPECIES = IntVector.SPECIES_MAX; - private static final VectorSpecies L_SPECIES = LongVector.SPECIES_MAX; - private static final VectorSpecies F_SPECIES = FloatVector.SPECIES_MAX; - private static final VectorSpecies D_SPECIES = DoubleVector.SPECIES_MAX; + private static final VectorSpecies B_SPECIES = VectorSpecies.ofLargestShape(byte.class); + private static final VectorSpecies S_SPECIES = VectorSpecies.ofLargestShape(short.class); + private static final VectorSpecies I_SPECIES = VectorSpecies.ofLargestShape(int.class); + private static final VectorSpecies L_SPECIES = VectorSpecies.ofLargestShape(long.class); + private static final VectorSpecies F_SPECIES = VectorSpecies.ofLargestShape(float.class); + private static final VectorSpecies D_SPECIES = VectorSpecies.ofLargestShape(double.class); private static final Generators RD = Generators.G; private static byte[] ba; @@ -90,8 +90,9 @@ public class VectorMaskCompareNotTest { dninf = new double[LENGTH]; mr = new boolean[LENGTH]; - Generator iGen = RD.uniformInts(Integer.MIN_VALUE, Integer.MAX_VALUE); - Generator lGen = RD.uniformLongs(Long.MIN_VALUE, Long.MAX_VALUE); + Generator iGen = RD.ints(); + Generator lGen = RD.longs(); + // Use uniform generators for floating point numbers not to generate NaN values. Generator fGen = RD.uniformFloats(Float.MIN_VALUE, Float.MAX_VALUE); Generator dGen = RD.uniformDoubles(Double.MIN_VALUE, Double.MAX_VALUE); for (int i = 0; i < LENGTH; i++) { @@ -118,11 +119,9 @@ public class VectorMaskCompareNotTest { public static int compareUnsigned(Number a, Number b) { if (a instanceof Byte) { - return Integer.compareUnsigned(Byte.toUnsignedInt(a.byteValue()), - Byte.toUnsignedInt(b.byteValue())); + return Integer.compareUnsigned(Byte.toUnsignedInt(a.byteValue()), Byte.toUnsignedInt(b.byteValue())); } else if (a instanceof Short) { - return Integer.compareUnsigned(Short.toUnsignedInt(a.shortValue()), - Short.toUnsignedInt(b.shortValue())); + return Integer.compareUnsigned(Short.toUnsignedInt(a.shortValue()), Short.toUnsignedInt(b.shortValue())); } else if (a instanceof Integer) { return Integer.compareUnsigned(a.intValue(), b.intValue()); } else if (a instanceof Long) { @@ -131,11 +130,14 @@ public static int compareUnsigned(Number a, Number b) { return 0; } - public static > void verifyResults(T a, T b, boolean r, - VectorOperators.Comparison op) { + public static > void verifyResults(T a, T b, boolean r, VectorOperators.Comparison op) { if (op == VectorOperators.EQ) { + // For floating point numbers, a is not NaN, b may be NaN. If b is NaN, + // a.compareTo(b) will return 1, 1 != 0 is true, r is expected to be true. Asserts.assertEquals(a.compareTo(b) != 0, r); } else if (op == VectorOperators.NE) { + // For floating point numbers, a is not NaN, b may be NaN. If b is NaN, + // a.compareTo(b) will return 1, 1 == 0 is false, r is expected to be false. Asserts.assertEquals(a.compareTo(b) == 0, r); } else if (op == VectorOperators.LE) { Asserts.assertEquals(a.compareTo(b) > 0, r); @@ -234,7 +236,7 @@ public static void testCompareMaskNotDouble(VectorOperators.Comparison op, doubl // Byte tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotByte() { testCompareMaskNotByte(VectorOperators.EQ, m -> m.not()); @@ -242,7 +244,7 @@ public static void testCompareEQMaskNotByte() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotByte() { testCompareMaskNotByte(VectorOperators.NE, m -> m.not()); @@ -250,7 +252,7 @@ public static void testCompareNEMaskNotByte() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotByte() { testCompareMaskNotByte(VectorOperators.LT, m -> m.not()); @@ -258,7 +260,7 @@ public static void testCompareLTMaskNotByte() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotByte() { testCompareMaskNotByte(VectorOperators.GT, m -> m.not()); @@ -266,7 +268,7 @@ public static void testCompareGTMaskNotByte() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotByte() { testCompareMaskNotByte(VectorOperators.LE, m -> m.not()); @@ -274,7 +276,7 @@ public static void testCompareLEMaskNotByte() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotByte() { testCompareMaskNotByte(VectorOperators.GE, m -> m.not()); @@ -282,7 +284,7 @@ public static void testCompareGEMaskNotByte() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotByte() { testCompareMaskNotByte(VectorOperators.ULT, m -> m.not()); @@ -290,7 +292,7 @@ public static void testCompareULTMaskNotByte() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotByte() { testCompareMaskNotByte(VectorOperators.UGT, m -> m.not()); @@ -298,7 +300,7 @@ public static void testCompareUGTMaskNotByte() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotByte() { testCompareMaskNotByte(VectorOperators.ULE, m -> m.not()); @@ -306,7 +308,7 @@ public static void testCompareULEMaskNotByte() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotByte() { testCompareMaskNotByte(VectorOperators.UGE, m -> m.not()); @@ -315,7 +317,7 @@ public static void testCompareUGEMaskNotByte() { // Short tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotShort() { testCompareMaskNotShort(VectorOperators.EQ, m -> m.not()); @@ -323,7 +325,7 @@ public static void testCompareEQMaskNotShort() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotShort() { testCompareMaskNotShort(VectorOperators.NE, m -> m.not()); @@ -331,7 +333,7 @@ public static void testCompareNEMaskNotShort() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotShort() { testCompareMaskNotShort(VectorOperators.LT, m -> m.not()); @@ -339,7 +341,7 @@ public static void testCompareLTMaskNotShort() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotShort() { testCompareMaskNotShort(VectorOperators.GT, m -> m.not()); @@ -347,7 +349,7 @@ public static void testCompareGTMaskNotShort() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotShort() { testCompareMaskNotShort(VectorOperators.LE, m -> m.not()); @@ -355,7 +357,7 @@ public static void testCompareLEMaskNotShort() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotShort() { testCompareMaskNotShort(VectorOperators.GE, m -> m.not()); @@ -363,7 +365,7 @@ public static void testCompareGEMaskNotShort() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotShort() { testCompareMaskNotShort(VectorOperators.ULT, m -> m.not()); @@ -371,7 +373,7 @@ public static void testCompareULTMaskNotShort() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotShort() { testCompareMaskNotShort(VectorOperators.UGT, m -> m.not()); @@ -379,7 +381,7 @@ public static void testCompareUGTMaskNotShort() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotShort() { testCompareMaskNotShort(VectorOperators.ULE, m -> m.not()); @@ -387,7 +389,7 @@ public static void testCompareULEMaskNotShort() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotShort() { testCompareMaskNotShort(VectorOperators.UGE, m -> m.not()); @@ -396,7 +398,7 @@ public static void testCompareUGEMaskNotShort() { // Int tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotInt() { testCompareMaskNotInt(VectorOperators.EQ, m -> m.not()); @@ -404,7 +406,7 @@ public static void testCompareEQMaskNotInt() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotInt() { testCompareMaskNotInt(VectorOperators.NE, m -> m.not()); @@ -412,7 +414,7 @@ public static void testCompareNEMaskNotInt() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLTMaskNotInt() { testCompareMaskNotInt(VectorOperators.LT, m -> m.not()); @@ -420,7 +422,7 @@ public static void testCompareLTMaskNotInt() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGTMaskNotInt() { testCompareMaskNotInt(VectorOperators.GT, m -> m.not()); @@ -428,7 +430,7 @@ public static void testCompareGTMaskNotInt() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareLEMaskNotInt() { testCompareMaskNotInt(VectorOperators.LE, m -> m.not()); @@ -436,7 +438,7 @@ public static void testCompareLEMaskNotInt() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareGEMaskNotInt() { testCompareMaskNotInt(VectorOperators.GE, m -> m.not()); @@ -444,7 +446,7 @@ public static void testCompareGEMaskNotInt() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULTMaskNotInt() { testCompareMaskNotInt(VectorOperators.ULT, m -> m.not()); @@ -452,7 +454,7 @@ public static void testCompareULTMaskNotInt() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGTMaskNotInt() { testCompareMaskNotInt(VectorOperators.UGT, m -> m.not()); @@ -460,7 +462,7 @@ public static void testCompareUGTMaskNotInt() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareULEMaskNotInt() { testCompareMaskNotInt(VectorOperators.ULE, m -> m.not()); @@ -468,7 +470,7 @@ public static void testCompareULEMaskNotInt() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareUGEMaskNotInt() { testCompareMaskNotInt(VectorOperators.UGE, m -> m.not()); @@ -477,87 +479,87 @@ public static void testCompareUGEMaskNotInt() { // Long tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareEQMaskNotLong() { testCompareMaskNotLong(VectorOperators.EQ, m -> m.not()); testCompareMaskNotLong(VectorOperators.EQ, m -> m.xor(L_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareNEMaskNotLong() { testCompareMaskNotLong(VectorOperators.NE, m -> m.not()); testCompareMaskNotLong(VectorOperators.NE, m -> m.xor(L_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareLTMaskNotLong() { testCompareMaskNotLong(VectorOperators.LT, m -> m.not()); testCompareMaskNotLong(VectorOperators.LT, m -> m.xor(L_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareGTMaskNotLong() { testCompareMaskNotLong(VectorOperators.GT, m -> m.not()); testCompareMaskNotLong(VectorOperators.GT, m -> m.xor(L_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareLEMaskNotLong() { testCompareMaskNotLong(VectorOperators.LE, m -> m.not()); testCompareMaskNotLong(VectorOperators.LE, m -> m.xor(L_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareGEMaskNotLong() { testCompareMaskNotLong(VectorOperators.GE, m -> m.not()); testCompareMaskNotLong(VectorOperators.GE, m -> m.xor(L_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareULTMaskNotLong() { testCompareMaskNotLong(VectorOperators.ULT, m -> m.not()); testCompareMaskNotLong(VectorOperators.ULT, m -> m.xor(L_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareUGTMaskNotLong() { testCompareMaskNotLong(VectorOperators.UGT, m -> m.not()); testCompareMaskNotLong(VectorOperators.UGT, m -> m.xor(L_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareULEMaskNotLong() { testCompareMaskNotLong(VectorOperators.ULE, m -> m.not()); testCompareMaskNotLong(VectorOperators.ULE, m -> m.xor(L_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareUGEMaskNotLong() { testCompareMaskNotLong(VectorOperators.UGE, m -> m.not()); testCompareMaskNotLong(VectorOperators.UGE, m -> m.xor(L_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloat() { testCompareMaskNotFloat(VectorOperators.EQ, fa, fb, m -> m.not()); @@ -565,7 +567,7 @@ public static void testCompareEQMaskNotFloat() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloat() { testCompareMaskNotFloat(VectorOperators.NE, fa, fb, m -> m.not()); @@ -573,7 +575,7 @@ public static void testCompareNEMaskNotFloat() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloatNaN() { testCompareMaskNotFloat(VectorOperators.EQ, fa, fnan, m -> m.not()); @@ -581,7 +583,7 @@ public static void testCompareEQMaskNotFloatNaN() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloatNaN() { testCompareMaskNotFloat(VectorOperators.NE, fa, fnan, m -> m.not()); @@ -589,7 +591,7 @@ public static void testCompareNEMaskNotFloatNaN() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloatPositiveInfinity() { testCompareMaskNotFloat(VectorOperators.EQ, fa, fpinf, m -> m.not()); @@ -597,7 +599,7 @@ public static void testCompareEQMaskNotFloatPositiveInfinity() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloatPositiveInfinity() { testCompareMaskNotFloat(VectorOperators.NE, fa, fpinf, m -> m.not()); @@ -605,7 +607,7 @@ public static void testCompareNEMaskNotFloatPositiveInfinity() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareEQMaskNotFloatNegativeInfinity() { testCompareMaskNotFloat(VectorOperators.EQ, fa, fninf, m -> m.not()); @@ -613,7 +615,7 @@ public static void testCompareEQMaskNotFloatNegativeInfinity() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) public static void testCompareNEMaskNotFloatNegativeInfinity() { testCompareMaskNotFloat(VectorOperators.NE, fa, fninf, m -> m.not()); @@ -621,64 +623,64 @@ public static void testCompareNEMaskNotFloatNegativeInfinity() { } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareEQMaskNotDouble() { testCompareMaskNotDouble(VectorOperators.EQ, da, db, m -> m.not()); testCompareMaskNotDouble(VectorOperators.EQ, da, db, m -> m.xor(D_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareNEMaskNotDouble() { testCompareMaskNotDouble(VectorOperators.NE, da, db, m -> m.not()); testCompareMaskNotDouble(VectorOperators.NE, da, db, m -> m.xor(D_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareEQMaskNotDoubleNaN() { testCompareMaskNotDouble(VectorOperators.EQ, da, dnan, m -> m.not()); testCompareMaskNotDouble(VectorOperators.EQ, da, dnan, m -> m.xor(D_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareNEMaskNotDoubleNaN() { testCompareMaskNotDouble(VectorOperators.NE, da, dnan, m -> m.not()); testCompareMaskNotDouble(VectorOperators.NE, da, dnan, m -> m.xor(D_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareEQMaskNotDoublePositiveInfinity() { testCompareMaskNotDouble(VectorOperators.EQ, da, dpinf, m -> m.not()); testCompareMaskNotDouble(VectorOperators.EQ, da, dpinf, m -> m.xor(D_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareNEMaskNotDoublePositiveInfinity() { testCompareMaskNotDouble(VectorOperators.NE, da, dpinf, m -> m.not()); testCompareMaskNotDouble(VectorOperators.NE, da, dpinf, m -> m.xor(D_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareEQMaskNotDoubleNegativeInfinity() { testCompareMaskNotDouble(VectorOperators.EQ, da, dninf, m -> m.not()); testCompareMaskNotDouble(VectorOperators.EQ, da, dninf, m -> m.xor(D_SPECIES.maskAll(true))); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, + applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) public static void testCompareNEMaskNotDoubleNegativeInfinity() { testCompareMaskNotDouble(VectorOperators.NE, da, dninf, m -> m.not()); testCompareMaskNotDouble(VectorOperators.NE, da, dninf, m -> m.xor(D_SPECIES.maskAll(true))); From f51bf722c3e576d6471e3ba6f2d09726ab1b8bbe Mon Sep 17 00:00:00 2001 From: erfang Date: Fri, 6 Jun 2025 10:29:25 +0000 Subject: [PATCH 07/12] Support negating unsigned comparison for BoolTest::mask Added a static method `negate_mask(mask btm)` into BoolTest class to negate both signed and unsigned comparison. --- src/hotspot/share/opto/subnode.hpp | 2 ++ src/hotspot/share/opto/vectornode.cpp | 2 +- src/hotspot/share/opto/vectornode.hpp | 3 --- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/opto/subnode.hpp b/src/hotspot/share/opto/subnode.hpp index 9a411f3d20e6b..a8c684c256e8e 100644 --- a/src/hotspot/share/opto/subnode.hpp +++ b/src/hotspot/share/opto/subnode.hpp @@ -329,6 +329,8 @@ struct BoolTest { // enum from above. mask commute( ) const { return mask("032147658"[_test]-'0'); } mask negate( ) const { return mask(_test^4); } + // Return the negative mask for the given mask, for both signed and unsigned comparison. + static mask negate_mask(mask btm) { return mask(btm^4); } bool is_canonical( ) const { return (_test == BoolTest::ne || _test == BoolTest::lt || _test == BoolTest::le || _test == BoolTest::overflow); } bool is_less( ) const { return _test == BoolTest::lt || _test == BoolTest::le; } bool is_greater( ) const { return _test == BoolTest::gt || _test == BoolTest::ge; } diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 51bd9f67a50c8..7c11b24af54a2 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2236,7 +2236,7 @@ Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { return nullptr; } - BoolTest::mask neg_cond = (BoolTest::mask) (((VectorMaskCmpNode*) in1)->get_negative_predicate()); + BoolTest::mask neg_cond = BoolTest::negate_mask(((VectorMaskCmpNode*) in1)->get_predicate()); ConINode* predicate_node = phase->intcon(neg_cond); const TypeVect* vt = in1->as_Vector()->vect_type(); Node* res = new VectorMaskCmpNode(neg_cond, in1->in(1), in1->in(2), diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 015029331fc74..d3aae937426c2 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1687,9 +1687,6 @@ class VectorMaskCmpNode : public VectorNode { } bool predicate_can_be_negated(); BoolTest::mask get_predicate() { return _predicate; } - // This is the same with BoolTest::negate(), but BoolTest doesn't support - // unsigned comparison. However _predicate can be unsigned comparison. - BoolTest::mask get_negative_predicate() { return BoolTest::mask(_predicate ^ 4); } #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; #endif // !PRODUCT From 5ebdc5726acf480bab89d2bb77032ab1b5c7fc24 Mon Sep 17 00:00:00 2001 From: erfang Date: Wed, 25 Jun 2025 10:03:42 +0000 Subject: [PATCH 08/12] Address more comments ATT. --- src/hotspot/share/opto/node.cpp | 6 +- src/hotspot/share/opto/subnode.hpp | 4 +- src/hotspot/share/opto/vectornode.cpp | 37 +- .../compiler/lib/ir_framework/IRNode.java | 20 +- .../checkattribute/parsing/RawIRNode.java | 4 +- .../vectorapi/VectorMaskCompareNotTest.java | 1134 ++++++++++++----- 6 files changed, 884 insertions(+), 321 deletions(-) diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index fd4a4f5c3f24f..706d1f1360c02 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -1217,11 +1217,7 @@ bool Node::has_special_unique_user() const { // See IfNode::fold_compares return true; } else if (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) { - // Condition for removing an unnecessary not() following a compare(...) - // operation. The predecessor of n (this XorV or XorVMask) may also be used - // by a useless VectorBox node which will later be eliminated by - // RemoveUseless. Return true to ensure that subgraph transformations are - // performed on n. + // Condition for XorVMask(VectorMaskCmp(x,y,cond), MaskAll(true)) ==> VectorMaskCmp(x,y,ncond) return true; } else { return false; diff --git a/src/hotspot/share/opto/subnode.hpp b/src/hotspot/share/opto/subnode.hpp index a8c684c256e8e..b112bdacd8742 100644 --- a/src/hotspot/share/opto/subnode.hpp +++ b/src/hotspot/share/opto/subnode.hpp @@ -328,9 +328,9 @@ struct BoolTest { // a simple char array where each element is the ASCII version of a 'mask' // enum from above. mask commute( ) const { return mask("032147658"[_test]-'0'); } - mask negate( ) const { return mask(_test^4); } + mask negate( ) const { return negate_mask(_test); } // Return the negative mask for the given mask, for both signed and unsigned comparison. - static mask negate_mask(mask btm) { return mask(btm^4); } + static mask negate_mask(mask btm) { return mask(btm ^ 4); } bool is_canonical( ) const { return (_test == BoolTest::ne || _test == BoolTest::lt || _test == BoolTest::le || _test == BoolTest::overflow); } bool is_less( ) const { return _test == BoolTest::lt || _test == BoolTest::le; } bool is_greater( ) const { return _test == BoolTest::gt || _test == BoolTest::ge; } diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index e1b04ba7f3368..83dc31198d834 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2197,6 +2197,10 @@ bool VectorMaskCmpNode::predicate_can_be_negated() { // => (VectorMaskCmp src1 src2 ncond) // (XorVMask (VectorMaskCmp src1 src2 cond) (MaskAll m1)) // => (VectorMaskCmp src1 src2 ncond) +// (XorV (VectorMaskCast (VectorMaskCmp src1 src2 cond)) (Replicate -1)) +// => (VectorMaskCast (VectorMaskCmp src1 src2 ncond)) +// (XorVMask (VectorMaskCast (VectorMaskCmp src1 src2 cond)) (MaskAll m1)) +// => (VectorMaskCast (VectorMaskCmp src1 src2 ncond)) // cond can be eq, ne, le, ge, lt, gt, ule, uge, ult and ugt. // ncond is the negative comparison of cond. // @@ -2217,35 +2221,34 @@ Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { } // XorV/XorVMask is commutative, swap VectorMaskCmp/VectorMaskCast to in1. - if (in2->Opcode() == Op_VectorMaskCmp || - (in2->Opcode() == Op_VectorMaskCast && in2->in(1)->Opcode() == Op_VectorMaskCmp)) { + if (VectorNode::is_all_ones_vector(in1)) { swap(in1, in2); } - const TypeVect* vector_mask_cast_vt = nullptr; - // in1 should be single used, otherwise the optimization may be unprofitable. - if (in1->Opcode() == Op_VectorMaskCast && in1->outcnt() == 1 && in1->in(1)->Opcode() == Op_VectorMaskCmp) { - vector_mask_cast_vt = in1->as_Vector()->vect_type(); + bool with_vector_mask_cast = false; + // VectorMaskCast and VectorMaskCmp should only have a single use, + // otherwise the optimization may be unprofitable. + if (in1->Opcode() == Op_VectorMaskCast) { + if (in1->outcnt() != 1) { + return nullptr; + } + with_vector_mask_cast = true; in1 = in1->in(1); } - if (in1->Opcode() != Op_VectorMaskCmp || - in1->outcnt() > 1 || - !((VectorMaskCmpNode*) in1)->predicate_can_be_negated() || + in1->outcnt() != 1 || + !(in1->as_VectorMaskCmp())->predicate_can_be_negated() || !VectorNode::is_all_ones_vector(in2)) { return nullptr; } - BoolTest::mask neg_cond = BoolTest::negate_mask(((VectorMaskCmpNode*) in1)->get_predicate()); + BoolTest::mask neg_cond = BoolTest::negate_mask((in1->as_VectorMaskCmp())->get_predicate()); ConINode* predicate_node = phase->intcon(neg_cond); const TypeVect* vt = in1->as_Vector()->vect_type(); - Node* res = new VectorMaskCmpNode(neg_cond, in1->in(1), in1->in(2), - predicate_node, vt); - if (vector_mask_cast_vt != nullptr) { - // We optimized out a VectorMaskCast, and in order to ensure type - // correctness, we need to regenerate one. VectorMaskCast will be encoded as - // a no-op (identity function) for types with the same size. - res = new VectorMaskCastNode(phase->transform(res), vector_mask_cast_vt); + Node* res = new VectorMaskCmpNode(neg_cond, in1->in(1), in1->in(2), predicate_node, vt); + if (with_vector_mask_cast) { + // We optimized out a VectorMaskCast, regenerate one to ensure type correctness. + res = new VectorMaskCastNode(phase->transform(res), vect_type()); } return res; } diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index f5a1ab6719d11..4edf798c3e604 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -2247,16 +2247,6 @@ public class IRNode { vectorNode(VECTOR_BLEND_D, "VectorBlend", TYPE_DOUBLE); } - public static final String VECTOR_MASK_CMP_B = VECTOR_PREFIX + "VECTOR_MASK_CMP_B" + POSTFIX; - static { - vectorNode(VECTOR_MASK_CMP_B, "VectorMaskCmp", TYPE_BYTE); - } - - public static final String VECTOR_MASK_CMP_S = VECTOR_PREFIX + "VECTOR_MASK_CMP_S" + POSTFIX; - static { - vectorNode(VECTOR_MASK_CMP_S, "VectorMaskCmp", TYPE_SHORT); - } - public static final String VECTOR_MASK_CMP_I = VECTOR_PREFIX + "VECTOR_MASK_CMP_I" + POSTFIX; static { vectorNode(VECTOR_MASK_CMP_I, "VectorMaskCmp", TYPE_INT); @@ -2277,6 +2267,11 @@ public class IRNode { vectorNode(VECTOR_MASK_CMP_D, "VectorMaskCmp", TYPE_DOUBLE); } + public static final String VECTOR_MASK_CMP = PREFIX + "VECTOR_MASK_CMP" + POSTFIX; + static { + beforeMatchingNameRegex(VECTOR_MASK_CMP, "VectorMaskCmp"); + } + public static final String VECTOR_CAST_B2S = VECTOR_PREFIX + "VECTOR_CAST_B2S" + POSTFIX; static { vectorNode(VECTOR_CAST_B2S, "VectorCastB2X", TYPE_SHORT); @@ -2687,6 +2682,11 @@ public class IRNode { vectorNode(XOR_VL, "XorV", TYPE_LONG); } + public static final String XOR_V = PREFIX + "XOR_V" + POSTFIX; + static { + beforeMatchingNameRegex(XOR_V, "XorV"); + } + public static final String XOR_V_MASK = PREFIX + "XOR_V_MASK" + POSTFIX; static { beforeMatchingNameRegex(XOR_V_MASK, "XorVMask"); diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/driver/irmatching/irrule/checkattribute/parsing/RawIRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/driver/irmatching/irrule/checkattribute/parsing/RawIRNode.java index ae06f2edd470a..bf3021a6868f1 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/driver/irmatching/irrule/checkattribute/parsing/RawIRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/driver/irmatching/irrule/checkattribute/parsing/RawIRNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -133,6 +133,6 @@ private String regexForVectorIRNode(String nodeRegex, VMInfo vmInfo, Comparison. } } String sizeRegex = IRNode.parseVectorNodeSize(size, type, vmInfo); - return nodeRegex.replaceAll(IRNode.IS_REPLACED, "vector([A-Za-z]|mask)<" + type + "," + sizeRegex + ">"); + return nodeRegex.replaceAll(IRNode.IS_REPLACED, "vector[A-Za-z]<" + type + "," + sizeRegex + ">"); } } diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java index 3d9d710721e6a..1c3dd542ef41a 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java @@ -23,8 +23,8 @@ package compiler.vectorapi; -import compiler.lib.ir_framework.*; import compiler.lib.generators.*; +import compiler.lib.ir_framework.*; import jdk.incubator.vector.*; import jdk.test.lib.Asserts; @@ -41,12 +41,20 @@ public class VectorMaskCompareNotTest { private static int LENGTH = 128; + private static final VectorSpecies B_SPECIES = VectorSpecies.ofLargestShape(byte.class); private static final VectorSpecies S_SPECIES = VectorSpecies.ofLargestShape(short.class); private static final VectorSpecies I_SPECIES = VectorSpecies.ofLargestShape(int.class); private static final VectorSpecies L_SPECIES = VectorSpecies.ofLargestShape(long.class); private static final VectorSpecies F_SPECIES = VectorSpecies.ofLargestShape(float.class); private static final VectorSpecies D_SPECIES = VectorSpecies.ofLargestShape(double.class); + + // Vector species for vector mask cast operation between int and long types, + // they must have the same number of elements. + // For other types, use a vector species of the specified width. + private static final VectorSpecies L_SPECIES_FOR_CAST = VectorSpecies.ofLargestShape(long.class); + private static final VectorSpecies I_SPECIES_FOR_CAST = VectorSpecies.of(int.class, VectorShape.forBitSize(L_SPECIES_FOR_CAST.vectorBitSize() / 2)); + private static final Generators RD = Generators.G; private static byte[] ba; @@ -55,6 +63,7 @@ public class VectorMaskCompareNotTest { private static short[] sb; private static int[] ia; private static int[] ib; + private static int[] ic; private static long[] la; private static long[] lb; private static float[] fa; @@ -76,6 +85,7 @@ public class VectorMaskCompareNotTest { sb = new short[LENGTH]; ia = new int[LENGTH]; ib = new int[LENGTH]; + ic = new int[LENGTH]; la = new long[LENGTH]; lb = new long[LENGTH]; fa = new float[LENGTH]; @@ -126,11 +136,12 @@ public static int compareUnsigned(Number a, Number b) { return Integer.compareUnsigned(a.intValue(), b.intValue()); } else if (a instanceof Long) { return Long.compareUnsigned(a.longValue(), b.longValue()); + } else { + throw new IllegalArgumentException("Unsupported type for unsigned comparison: " + a.getClass() + ", " + b.getClass()); } - return 0; } - public static > void verifyResults(T a, T b, boolean r, VectorOperators.Comparison op) { + public static > void compareResults(T a, T b, boolean r, VectorOperators.Comparison op) { if (op == VectorOperators.EQ) { // For floating point numbers, a is not NaN, b may be NaN. If b is NaN, // a.compareTo(b) will return 1, 1 != 0 is true, r is expected to be true. @@ -155,541 +166,1094 @@ public static > void verifyResults(T a, T b, bo Asserts.assertEquals(compareUnsigned(a, b) >= 0, r); } else if (op == VectorOperators.UGT) { Asserts.assertEquals(compareUnsigned(a, b) <= 0, r); + } else { + throw new IllegalArgumentException("Unknown comparison operator: " + op); + } + } + + @DontInline + public static void verifyResultsByte(VectorSpecies vs, VectorOperators.Comparison op) { + for (int i = 0; i < vs.length(); i++) { + compareResults(ba[i], bb[i], mr[i], op); + } + } + + @DontInline + public static void verifyResultsShort(VectorSpecies vs, VectorOperators.Comparison op) { + for (int i = 0; i < vs.length(); i++) { + compareResults(sa[i], sb[i], mr[i], op); + } + } + + @DontInline + public static void verifyResultsInt(VectorSpecies vs, VectorOperators.Comparison op) { + for (int i = 0; i < vs.length(); i++) { + compareResults(ia[i], ib[i], mr[i], op); + } + } + + @DontInline + public static void verifyResultsLong(VectorSpecies vs, VectorOperators.Comparison op) { + for (int i = 0; i < vs.length(); i++) { + compareResults(la[i], lb[i], mr[i], op); + } + } + + @DontInline + public static void verifyResultsFloat(VectorSpecies vs, VectorOperators.Comparison op, float[] a, float[] b) { + for (int i = 0; i < vs.length(); i++) { + compareResults(a[i], b[i], mr[i], op); + } + } + + @DontInline + public static void verifyResultsDouble(VectorSpecies vs, VectorOperators.Comparison op, double[] a, double[] b) { + for (int i = 0; i < vs.length(); i++) { + compareResults(a[i], b[i], mr[i], op); } } interface VectorMaskOperator { - public VectorMask apply(VectorMask m1); + public VectorMask apply(VectorMask m); } @ForceInline - public static void testCompareMaskNotByte(VectorOperators.Comparison op, VectorMaskOperator func) { - ByteVector av = ByteVector.fromArray(B_SPECIES, ba, 0); - ByteVector bv = ByteVector.fromArray(B_SPECIES, bb, 0); - VectorMask m1 = av.compare(op, bv); - func.apply(m1).intoArray(mr, 0); - - for (int i = 0; i < B_SPECIES.length(); i++) { - verifyResults(ba[i], bb[i], mr[i], op); - } + public static void testCompareMaskNotByte(VectorSpecies vs, VectorOperators.Comparison op, VectorMaskOperator func) { + ByteVector av = ByteVector.fromArray(vs, ba, 0); + ByteVector bv = ByteVector.fromArray(vs, bb, 0); + VectorMask m = av.compare(op, bv); + func.apply(m).intoArray(mr, 0); } @ForceInline - public static void testCompareMaskNotShort(VectorOperators.Comparison op, VectorMaskOperator func) { - ShortVector av = ShortVector.fromArray(S_SPECIES, sa, 0); - ShortVector bv = ShortVector.fromArray(S_SPECIES, sb, 0); - VectorMask m1 = av.compare(op, bv); - func.apply(m1).intoArray(mr, 0); - - for (int i = 0; i < S_SPECIES.length(); i++) { - verifyResults(sa[i], sb[i], mr[i], op); - } + public static void testCompareMaskNotShort(VectorSpecies vs, VectorOperators.Comparison op, VectorMaskOperator func) { + ShortVector av = ShortVector.fromArray(vs, sa, 0); + ShortVector bv = ShortVector.fromArray(vs, sb, 0); + VectorMask m = av.compare(op, bv); + func.apply(m).intoArray(mr, 0); } @ForceInline - public static void testCompareMaskNotInt(VectorOperators.Comparison op, VectorMaskOperator func) { - IntVector av = IntVector.fromArray(I_SPECIES, ia, 0); - IntVector bv = IntVector.fromArray(I_SPECIES, ib, 0); - VectorMask m1 = av.compare(op, bv); - func.apply(m1).intoArray(mr, 0); - - for (int i = 0; i < I_SPECIES.length(); i++) { - verifyResults(ia[i], ib[i], mr[i], op); - } + public static void testCompareMaskNotInt(VectorSpecies vs, VectorOperators.Comparison op, VectorMaskOperator func) { + IntVector av = IntVector.fromArray(vs, ia, 0); + IntVector bv = IntVector.fromArray(vs, ib, 0); + VectorMask m = av.compare(op, bv); + func.apply(m).intoArray(mr, 0); } @ForceInline - public static void testCompareMaskNotLong(VectorOperators.Comparison op, VectorMaskOperator func) { - LongVector av = LongVector.fromArray(L_SPECIES, la, 0); - LongVector bv = LongVector.fromArray(L_SPECIES, lb, 0); - VectorMask m1 = av.compare(op, bv); - func.apply(m1).intoArray(mr, 0); - - for (int i = 0; i < L_SPECIES.length(); i++) { - verifyResults(la[i], lb[i], mr[i], op); - } + public static void testCompareMaskNotLong(VectorSpecies vs, VectorOperators.Comparison op, VectorMaskOperator func) { + LongVector av = LongVector.fromArray(vs, la, 0); + LongVector bv = LongVector.fromArray(vs, lb, 0); + VectorMask m = av.compare(op, bv); + func.apply(m).intoArray(mr, 0); } @ForceInline - public static void testCompareMaskNotFloat(VectorOperators.Comparison op, float[] a, float[] b, VectorMaskOperator func) { - FloatVector av = FloatVector.fromArray(F_SPECIES, a, 0); - FloatVector bv = FloatVector.fromArray(F_SPECIES, b, 0); - VectorMask m1 = av.compare(op, bv); - func.apply(m1).intoArray(mr, 0); - - for (int i = 0; i < F_SPECIES.length(); i++) { - verifyResults(a[i], b[i], mr[i], op); - } + public static void testCompareMaskNotFloat(VectorSpecies vs, VectorOperators.Comparison op, float[] a, float[] b, VectorMaskOperator func) { + FloatVector av = FloatVector.fromArray(vs, a, 0); + FloatVector bv = FloatVector.fromArray(vs, b, 0); + VectorMask m = av.compare(op, bv); + func.apply(m).intoArray(mr, 0); } @ForceInline - public static void testCompareMaskNotDouble(VectorOperators.Comparison op, double[] a, double[] b, VectorMaskOperator func) { - DoubleVector av = DoubleVector.fromArray(D_SPECIES, a, 0); - DoubleVector bv = DoubleVector.fromArray(D_SPECIES, b, 0); - VectorMask m1 = av.compare(op, bv); - func.apply(m1).intoArray(mr, 0); - - for (int i = 0; i < D_SPECIES.length(); i++) { - verifyResults(a[i], b[i], mr[i], op); - } + public static void testCompareMaskNotDouble(VectorSpecies vs, VectorOperators.Comparison op, double[] a, double[] b, VectorMaskOperator func) { + DoubleVector av = DoubleVector.fromArray(vs, a, 0); + DoubleVector bv = DoubleVector.fromArray(vs, b, 0); + VectorMask m = av.compare(op, bv); + func.apply(m).intoArray(mr, 0); } // Byte tests + @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareEQMaskNotByte() { - testCompareMaskNotByte(VectorOperators.EQ, m -> m.not()); - testCompareMaskNotByte(VectorOperators.EQ, m -> m.xor(B_SPECIES.maskAll(true))); + testCompareMaskNotByte(B_SPECIES, VectorOperators.EQ, (m) -> { return m.not(); }); + verifyResultsByte(B_SPECIES, VectorOperators.EQ); + testCompareMaskNotByte(B_SPECIES, VectorOperators.EQ, (m) -> { return B_SPECIES.maskAll(true).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.EQ); + + testCompareMaskNotByte(ByteVector.SPECIES_64, VectorOperators.EQ, (m) -> { return m.cast(ShortVector.SPECIES_128).not(); }); + verifyResultsByte(ByteVector.SPECIES_64, VectorOperators.EQ); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareNEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.NE, m -> m.not()); - testCompareMaskNotByte(VectorOperators.NE, m -> m.xor(B_SPECIES.maskAll(true))); + testCompareMaskNotByte(B_SPECIES, VectorOperators.NE, (m) -> { return m.not(); }); + verifyResultsByte(B_SPECIES, VectorOperators.NE); + testCompareMaskNotByte(B_SPECIES, VectorOperators.NE, (m) -> { return B_SPECIES.maskAll(true).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.NE); + + testCompareMaskNotByte(ByteVector.SPECIES_64, VectorOperators.NE, (m) -> { return m.cast(ShortVector.SPECIES_128).not(); }); + verifyResultsByte(ByteVector.SPECIES_64, VectorOperators.NE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareLTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.LT, m -> m.not()); - testCompareMaskNotByte(VectorOperators.LT, m -> m.xor(B_SPECIES.maskAll(true))); + testCompareMaskNotByte(B_SPECIES, VectorOperators.LT, (m) -> { return m.not(); }); + verifyResultsByte(B_SPECIES, VectorOperators.LT); + testCompareMaskNotByte(B_SPECIES, VectorOperators.LT, (m) -> { return B_SPECIES.maskAll(true).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.LT); + + testCompareMaskNotByte(ByteVector.SPECIES_64, VectorOperators.LT, (m) -> { return m.cast(ShortVector.SPECIES_128).not(); }); + verifyResultsByte(ByteVector.SPECIES_64, VectorOperators.LT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareGTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.GT, m -> m.not()); - testCompareMaskNotByte(VectorOperators.GT, m -> m.xor(B_SPECIES.maskAll(true))); + testCompareMaskNotByte(B_SPECIES, VectorOperators.GT, (m) -> { return m.not(); }); + verifyResultsByte(B_SPECIES, VectorOperators.GT); + testCompareMaskNotByte(B_SPECIES, VectorOperators.GT, (m) -> { return B_SPECIES.maskAll(true).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.GT); + + testCompareMaskNotByte(ByteVector.SPECIES_64, VectorOperators.GT, (m) -> { return m.cast(ShortVector.SPECIES_128).not(); }); + verifyResultsByte(ByteVector.SPECIES_64, VectorOperators.GT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareLEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.LE, m -> m.not()); - testCompareMaskNotByte(VectorOperators.LE, m -> m.xor(B_SPECIES.maskAll(true))); + testCompareMaskNotByte(B_SPECIES, VectorOperators.LE, (m) -> { return m.not(); }); + verifyResultsByte(B_SPECIES, VectorOperators.LE); + testCompareMaskNotByte(B_SPECIES, VectorOperators.LE, (m) -> { return B_SPECIES.maskAll(true).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.LE); + + testCompareMaskNotByte(ByteVector.SPECIES_64, VectorOperators.LE, (m) -> { return m.cast(ShortVector.SPECIES_128).not(); }); + verifyResultsByte(ByteVector.SPECIES_64, VectorOperators.LE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareGEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.GE, m -> m.not()); - testCompareMaskNotByte(VectorOperators.GE, m -> m.xor(B_SPECIES.maskAll(true))); + testCompareMaskNotByte(B_SPECIES, VectorOperators.GE, (m) -> { return m.not(); }); + verifyResultsByte(B_SPECIES, VectorOperators.GE); + testCompareMaskNotByte(B_SPECIES, VectorOperators.GE, (m) -> { return B_SPECIES.maskAll(true).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.GE); + + testCompareMaskNotByte(ByteVector.SPECIES_64, VectorOperators.GE, (m) -> { return m.cast(ShortVector.SPECIES_128).not(); }); + verifyResultsByte(ByteVector.SPECIES_64, VectorOperators.GE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareULTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.ULT, m -> m.not()); - testCompareMaskNotByte(VectorOperators.ULT, m -> m.xor(B_SPECIES.maskAll(true))); + testCompareMaskNotByte(B_SPECIES, VectorOperators.ULT, (m) -> { return m.not(); }); + verifyResultsByte(B_SPECIES, VectorOperators.ULT); + testCompareMaskNotByte(B_SPECIES, VectorOperators.ULT, (m) -> { return B_SPECIES.maskAll(true).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.ULT); + + testCompareMaskNotByte(ByteVector.SPECIES_64, VectorOperators.ULT, (m) -> { return m.cast(ShortVector.SPECIES_128).not(); }); + verifyResultsByte(ByteVector.SPECIES_64, VectorOperators.ULT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareUGTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.UGT, m -> m.not()); - testCompareMaskNotByte(VectorOperators.UGT, m -> m.xor(B_SPECIES.maskAll(true))); + testCompareMaskNotByte(B_SPECIES, VectorOperators.UGT, (m) -> { return m.not(); }); + verifyResultsByte(B_SPECIES, VectorOperators.UGT); + testCompareMaskNotByte(B_SPECIES, VectorOperators.UGT, (m) -> { return B_SPECIES.maskAll(true).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.UGT); + + testCompareMaskNotByte(ByteVector.SPECIES_64, VectorOperators.UGT, (m) -> { return m.cast(ShortVector.SPECIES_128).not(); }); + verifyResultsByte(ByteVector.SPECIES_64, VectorOperators.UGT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareULEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.ULE, m -> m.not()); - testCompareMaskNotByte(VectorOperators.ULE, m -> m.xor(B_SPECIES.maskAll(true))); + testCompareMaskNotByte(B_SPECIES, VectorOperators.ULE, (m) -> { return m.not(); }); + verifyResultsByte(B_SPECIES, VectorOperators.ULE); + testCompareMaskNotByte(B_SPECIES, VectorOperators.ULE, (m) -> { return B_SPECIES.maskAll(true).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.ULE); + + testCompareMaskNotByte(ByteVector.SPECIES_64, VectorOperators.ULE, (m) -> { return m.cast(ShortVector.SPECIES_128).not(); }); + verifyResultsByte(ByteVector.SPECIES_64, VectorOperators.ULE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VB, "= 0", IRNode.VECTOR_MASK_CMP_B, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareUGEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.UGE, m -> m.not()); - testCompareMaskNotByte(VectorOperators.UGE, m -> m.xor(B_SPECIES.maskAll(true))); + testCompareMaskNotByte(B_SPECIES, VectorOperators.UGE, (m) -> { return m.not(); }); + verifyResultsByte(B_SPECIES, VectorOperators.UGE); + testCompareMaskNotByte(B_SPECIES, VectorOperators.UGE, (m) -> { return B_SPECIES.maskAll(true).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.UGE); + + testCompareMaskNotByte(ByteVector.SPECIES_64, VectorOperators.UGE, (m) -> { return m.cast(ShortVector.SPECIES_128).not(); }); + verifyResultsByte(ByteVector.SPECIES_64, VectorOperators.UGE); } // Short tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareEQMaskNotShort() { - testCompareMaskNotShort(VectorOperators.EQ, m -> m.not()); - testCompareMaskNotShort(VectorOperators.EQ, m -> m.xor(S_SPECIES.maskAll(true))); + testCompareMaskNotShort(S_SPECIES, VectorOperators.EQ, (m) -> { return m.not(); }); + verifyResultsShort(S_SPECIES, VectorOperators.EQ); + testCompareMaskNotShort(S_SPECIES, VectorOperators.EQ, (m) -> { return S_SPECIES.maskAll(true).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.EQ); + + testCompareMaskNotShort(ShortVector.SPECIES_64, VectorOperators.EQ, (m) -> { return IntVector.SPECIES_128.maskAll(true).xor(m.cast(IntVector.SPECIES_128)); }); + verifyResultsShort(ShortVector.SPECIES_64, VectorOperators.EQ); + testCompareMaskNotShort(ShortVector.SPECIES_128, VectorOperators.EQ, (m) -> { return m.cast(ByteVector.SPECIES_64).not(); }); + verifyResultsShort(ShortVector.SPECIES_128, VectorOperators.EQ); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareNEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.NE, m -> m.not()); - testCompareMaskNotShort(VectorOperators.NE, m -> m.xor(S_SPECIES.maskAll(true))); + testCompareMaskNotShort(S_SPECIES, VectorOperators.NE, (m) -> { return m.not(); }); + verifyResultsShort(S_SPECIES, VectorOperators.NE); + testCompareMaskNotShort(S_SPECIES, VectorOperators.NE, (m) -> { return S_SPECIES.maskAll(true).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.NE); + + testCompareMaskNotShort(ShortVector.SPECIES_64, VectorOperators.NE, (m) -> { return IntVector.SPECIES_128.maskAll(true).xor(m.cast(IntVector.SPECIES_128)); }); + verifyResultsShort(ShortVector.SPECIES_64, VectorOperators.NE); + testCompareMaskNotShort(ShortVector.SPECIES_128, VectorOperators.NE, (m) -> { return m.cast(ByteVector.SPECIES_64).not(); }); + verifyResultsShort(ShortVector.SPECIES_128, VectorOperators.NE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareLTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.LT, m -> m.not()); - testCompareMaskNotShort(VectorOperators.LT, m -> m.xor(S_SPECIES.maskAll(true))); + testCompareMaskNotShort(S_SPECIES, VectorOperators.LT, (m) -> { return m.not(); }); + verifyResultsShort(S_SPECIES, VectorOperators.LT); + testCompareMaskNotShort(S_SPECIES, VectorOperators.LT, (m) -> { return S_SPECIES.maskAll(true).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.LT); + + testCompareMaskNotShort(ShortVector.SPECIES_64, VectorOperators.LT, (m) -> { return IntVector.SPECIES_128.maskAll(true).xor(m.cast(IntVector.SPECIES_128)); }); + verifyResultsShort(ShortVector.SPECIES_64, VectorOperators.LT); + testCompareMaskNotShort(ShortVector.SPECIES_128, VectorOperators.LT, (m) -> { return m.cast(ByteVector.SPECIES_64).not(); }); + verifyResultsShort(ShortVector.SPECIES_128, VectorOperators.LT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareGTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.GT, m -> m.not()); - testCompareMaskNotShort(VectorOperators.GT, m -> m.xor(S_SPECIES.maskAll(true))); + testCompareMaskNotShort(S_SPECIES, VectorOperators.GT, (m) -> { return m.not(); }); + verifyResultsShort(S_SPECIES, VectorOperators.GT); + testCompareMaskNotShort(S_SPECIES, VectorOperators.GT, (m) -> { return S_SPECIES.maskAll(true).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.GT); + + testCompareMaskNotShort(ShortVector.SPECIES_64, VectorOperators.GT, (m) -> { return IntVector.SPECIES_128.maskAll(true).xor(m.cast(IntVector.SPECIES_128)); }); + verifyResultsShort(ShortVector.SPECIES_64, VectorOperators.GT); + testCompareMaskNotShort(ShortVector.SPECIES_128, VectorOperators.GT, (m) -> { return m.cast(ByteVector.SPECIES_64).not(); }); + verifyResultsShort(ShortVector.SPECIES_128, VectorOperators.GT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareLEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.LE, m -> m.not()); - testCompareMaskNotShort(VectorOperators.LE, m -> m.xor(S_SPECIES.maskAll(true))); + testCompareMaskNotShort(S_SPECIES, VectorOperators.LE, (m) -> { return m.not(); }); + verifyResultsShort(S_SPECIES, VectorOperators.LE); + testCompareMaskNotShort(S_SPECIES, VectorOperators.LE, (m) -> { return S_SPECIES.maskAll(true).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.LE); + + testCompareMaskNotShort(ShortVector.SPECIES_64, VectorOperators.LE, (m) -> { return IntVector.SPECIES_128.maskAll(true).xor(m.cast(IntVector.SPECIES_128)); }); + verifyResultsShort(ShortVector.SPECIES_64, VectorOperators.LE); + testCompareMaskNotShort(ShortVector.SPECIES_128, VectorOperators.LE, (m) -> { return m.cast(ByteVector.SPECIES_64).not(); }); + verifyResultsShort(ShortVector.SPECIES_128, VectorOperators.LE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareGEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.GE, m -> m.not()); - testCompareMaskNotShort(VectorOperators.GE, m -> m.xor(S_SPECIES.maskAll(true))); + testCompareMaskNotShort(S_SPECIES, VectorOperators.GE, (m) -> { return m.not(); }); + verifyResultsShort(S_SPECIES, VectorOperators.GE); + testCompareMaskNotShort(S_SPECIES, VectorOperators.GE, (m) -> { return S_SPECIES.maskAll(true).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.GE); + + testCompareMaskNotShort(ShortVector.SPECIES_64, VectorOperators.GE, (m) -> { return IntVector.SPECIES_128.maskAll(true).xor(m.cast(IntVector.SPECIES_128)); }); + verifyResultsShort(ShortVector.SPECIES_64, VectorOperators.GE); + testCompareMaskNotShort(ShortVector.SPECIES_128, VectorOperators.GE, (m) -> { return m.cast(ByteVector.SPECIES_64).not(); }); + verifyResultsShort(ShortVector.SPECIES_128, VectorOperators.GE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareULTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.ULT, m -> m.not()); - testCompareMaskNotShort(VectorOperators.ULT, m -> m.xor(S_SPECIES.maskAll(true))); + testCompareMaskNotShort(S_SPECIES, VectorOperators.ULT, (m) -> { return m.not(); }); + verifyResultsShort(S_SPECIES, VectorOperators.ULT); + testCompareMaskNotShort(S_SPECIES, VectorOperators.ULT, (m) -> { return S_SPECIES.maskAll(true).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.ULT); + + testCompareMaskNotShort(ShortVector.SPECIES_64, VectorOperators.ULT, (m) -> { return IntVector.SPECIES_128.maskAll(true).xor(m.cast(IntVector.SPECIES_128)); }); + verifyResultsShort(ShortVector.SPECIES_64, VectorOperators.ULT); + testCompareMaskNotShort(ShortVector.SPECIES_128, VectorOperators.ULT, (m) -> { return m.cast(ByteVector.SPECIES_64).not(); }); + verifyResultsShort(ShortVector.SPECIES_128, VectorOperators.ULT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareUGTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.UGT, m -> m.not()); - testCompareMaskNotShort(VectorOperators.UGT, m -> m.xor(S_SPECIES.maskAll(true))); + testCompareMaskNotShort(S_SPECIES, VectorOperators.UGT, (m) -> { return m.not(); }); + verifyResultsShort(S_SPECIES, VectorOperators.UGT); + testCompareMaskNotShort(S_SPECIES, VectorOperators.UGT, (m) -> { return S_SPECIES.maskAll(true).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.UGT); + + testCompareMaskNotShort(ShortVector.SPECIES_64, VectorOperators.UGT, (m) -> { return IntVector.SPECIES_128.maskAll(true).xor(m.cast(IntVector.SPECIES_128)); }); + verifyResultsShort(ShortVector.SPECIES_64, VectorOperators.UGT); + testCompareMaskNotShort(ShortVector.SPECIES_128, VectorOperators.UGT, (m) -> { return m.cast(ByteVector.SPECIES_64).not(); }); + verifyResultsShort(ShortVector.SPECIES_128, VectorOperators.UGT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareULEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.ULE, m -> m.not()); - testCompareMaskNotShort(VectorOperators.ULE, m -> m.xor(S_SPECIES.maskAll(true))); + testCompareMaskNotShort(S_SPECIES, VectorOperators.ULE, (m) -> { return m.not(); }); + verifyResultsShort(S_SPECIES, VectorOperators.ULE); + testCompareMaskNotShort(S_SPECIES, VectorOperators.ULE, (m) -> { return S_SPECIES.maskAll(true).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.ULE); + + testCompareMaskNotShort(ShortVector.SPECIES_64, VectorOperators.ULE, (m) -> { return IntVector.SPECIES_128.maskAll(true).xor(m.cast(IntVector.SPECIES_128)); }); + verifyResultsShort(ShortVector.SPECIES_64, VectorOperators.ULE); + testCompareMaskNotShort(ShortVector.SPECIES_128, VectorOperators.ULE, (m) -> { return m.cast(ByteVector.SPECIES_64).not(); }); + verifyResultsShort(ShortVector.SPECIES_128, VectorOperators.ULE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VS, "= 0", IRNode.VECTOR_MASK_CMP_S, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareUGEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.UGE, m -> m.not()); - testCompareMaskNotShort(VectorOperators.UGE, m -> m.xor(S_SPECIES.maskAll(true))); + testCompareMaskNotShort(S_SPECIES, VectorOperators.UGE, (m) -> { return m.not(); }); + verifyResultsShort(S_SPECIES, VectorOperators.UGE); + testCompareMaskNotShort(S_SPECIES, VectorOperators.UGE, (m) -> { return S_SPECIES.maskAll(true).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.UGE); + + testCompareMaskNotShort(ShortVector.SPECIES_64, VectorOperators.UGE, (m) -> { return IntVector.SPECIES_128.maskAll(true).xor(m.cast(IntVector.SPECIES_128)); }); + verifyResultsShort(ShortVector.SPECIES_64, VectorOperators.UGE); + testCompareMaskNotShort(ShortVector.SPECIES_128, VectorOperators.UGE, (m) -> { return m.cast(ByteVector.SPECIES_64).not(); }); + verifyResultsShort(ShortVector.SPECIES_128, VectorOperators.UGE); } // Int tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareEQMaskNotInt() { - testCompareMaskNotInt(VectorOperators.EQ, m -> m.not()); - testCompareMaskNotInt(VectorOperators.EQ, m -> m.xor(I_SPECIES.maskAll(true))); + testCompareMaskNotInt(I_SPECIES, VectorOperators.EQ, (m) -> { return m.not(); }); + verifyResultsInt(I_SPECIES, VectorOperators.EQ); + testCompareMaskNotInt(I_SPECIES, VectorOperators.EQ, (m) -> { return I_SPECIES.maskAll(true).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.EQ); + + testCompareMaskNotInt(I_SPECIES_FOR_CAST, VectorOperators.EQ, (m) -> { return L_SPECIES_FOR_CAST.maskAll(true).xor(m.cast(L_SPECIES_FOR_CAST)); }); + verifyResultsInt(I_SPECIES_FOR_CAST, VectorOperators.EQ); + testCompareMaskNotInt(IntVector.SPECIES_128, VectorOperators.EQ, (m) -> { return m.cast(ShortVector.SPECIES_64).not(); }); + verifyResultsInt(IntVector.SPECIES_128, VectorOperators.EQ); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareNEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.NE, m -> m.not()); - testCompareMaskNotInt(VectorOperators.NE, m -> m.xor(I_SPECIES.maskAll(true))); + testCompareMaskNotInt(I_SPECIES, VectorOperators.NE, (m) -> { return m.not(); }); + verifyResultsInt(I_SPECIES, VectorOperators.NE); + testCompareMaskNotInt(I_SPECIES, VectorOperators.NE, (m) -> { return I_SPECIES.maskAll(true).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.NE); + + testCompareMaskNotInt(I_SPECIES_FOR_CAST, VectorOperators.NE, (m) -> { return L_SPECIES_FOR_CAST.maskAll(true).xor(m.cast(L_SPECIES_FOR_CAST)); }); + verifyResultsInt(I_SPECIES_FOR_CAST, VectorOperators.NE); + testCompareMaskNotInt(IntVector.SPECIES_128, VectorOperators.NE, (m) -> { return m.cast(ShortVector.SPECIES_64).not(); }); + verifyResultsInt(IntVector.SPECIES_128, VectorOperators.NE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareLTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.LT, m -> m.not()); - testCompareMaskNotInt(VectorOperators.LT, m -> m.xor(I_SPECIES.maskAll(true))); + testCompareMaskNotInt(I_SPECIES, VectorOperators.LT, (m) -> { return m.not(); }); + verifyResultsInt(I_SPECIES, VectorOperators.LT); + testCompareMaskNotInt(I_SPECIES, VectorOperators.LT, (m) -> { return I_SPECIES.maskAll(true).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.LT); + + testCompareMaskNotInt(I_SPECIES_FOR_CAST, VectorOperators.LT, (m) -> { return L_SPECIES_FOR_CAST.maskAll(true).xor(m.cast(L_SPECIES_FOR_CAST)); }); + verifyResultsInt(I_SPECIES_FOR_CAST, VectorOperators.LT); + testCompareMaskNotInt(IntVector.SPECIES_128, VectorOperators.LT, (m) -> { return m.cast(ShortVector.SPECIES_64).not(); }); + verifyResultsInt(IntVector.SPECIES_128, VectorOperators.LT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareGTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.GT, m -> m.not()); - testCompareMaskNotInt(VectorOperators.GT, m -> m.xor(I_SPECIES.maskAll(true))); + testCompareMaskNotInt(I_SPECIES, VectorOperators.GT, (m) -> { return m.not(); }); + verifyResultsInt(I_SPECIES, VectorOperators.GT); + testCompareMaskNotInt(I_SPECIES, VectorOperators.GT, (m) -> { return I_SPECIES.maskAll(true).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.GT); + + testCompareMaskNotInt(I_SPECIES_FOR_CAST, VectorOperators.GT, (m) -> { return L_SPECIES_FOR_CAST.maskAll(true).xor(m.cast(L_SPECIES_FOR_CAST)); }); + verifyResultsInt(I_SPECIES_FOR_CAST, VectorOperators.GT); + testCompareMaskNotInt(IntVector.SPECIES_128, VectorOperators.GT, (m) -> { return m.cast(ShortVector.SPECIES_64).not(); }); + verifyResultsInt(IntVector.SPECIES_128, VectorOperators.GT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareLEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.LE, m -> m.not()); - testCompareMaskNotInt(VectorOperators.LE, m -> m.xor(I_SPECIES.maskAll(true))); + testCompareMaskNotInt(I_SPECIES, VectorOperators.LE, (m) -> { return m.not(); }); + verifyResultsInt(I_SPECIES, VectorOperators.LE); + testCompareMaskNotInt(I_SPECIES, VectorOperators.LE, (m) -> { return I_SPECIES.maskAll(true).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.LE); + + testCompareMaskNotInt(I_SPECIES_FOR_CAST, VectorOperators.LE, (m) -> { return L_SPECIES_FOR_CAST.maskAll(true).xor(m.cast(L_SPECIES_FOR_CAST)); }); + verifyResultsInt(I_SPECIES_FOR_CAST, VectorOperators.LE); + testCompareMaskNotInt(IntVector.SPECIES_128, VectorOperators.LE, (m) -> { return m.cast(ShortVector.SPECIES_64).not(); }); + verifyResultsInt(IntVector.SPECIES_128, VectorOperators.LE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareGEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.GE, m -> m.not()); - testCompareMaskNotInt(VectorOperators.GE, m -> m.xor(I_SPECIES.maskAll(true))); + testCompareMaskNotInt(I_SPECIES, VectorOperators.GE, (m) -> { return m.not(); }); + verifyResultsInt(I_SPECIES, VectorOperators.GE); + testCompareMaskNotInt(I_SPECIES, VectorOperators.GE, (m) -> { return I_SPECIES.maskAll(true).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.GE); + + testCompareMaskNotInt(I_SPECIES_FOR_CAST, VectorOperators.GE, (m) -> { return L_SPECIES_FOR_CAST.maskAll(true).xor(m.cast(L_SPECIES_FOR_CAST)); }); + verifyResultsInt(I_SPECIES_FOR_CAST, VectorOperators.GE); + testCompareMaskNotInt(IntVector.SPECIES_128, VectorOperators.GE, (m) -> { return m.cast(ShortVector.SPECIES_64).not(); }); + verifyResultsInt(IntVector.SPECIES_128, VectorOperators.GE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareULTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.ULT, m -> m.not()); - testCompareMaskNotInt(VectorOperators.ULT, m -> m.xor(I_SPECIES.maskAll(true))); + testCompareMaskNotInt(I_SPECIES, VectorOperators.ULT, (m) -> { return m.not(); }); + verifyResultsInt(I_SPECIES, VectorOperators.ULT); + testCompareMaskNotInt(I_SPECIES, VectorOperators.ULT, (m) -> { return I_SPECIES.maskAll(true).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.ULT); + + testCompareMaskNotInt(I_SPECIES_FOR_CAST, VectorOperators.ULT, (m) -> { return L_SPECIES_FOR_CAST.maskAll(true).xor(m.cast(L_SPECIES_FOR_CAST)); }); + verifyResultsInt(I_SPECIES_FOR_CAST, VectorOperators.ULT); + testCompareMaskNotInt(IntVector.SPECIES_128, VectorOperators.ULT, (m) -> { return m.cast(ShortVector.SPECIES_64).not(); }); + verifyResultsInt(IntVector.SPECIES_128, VectorOperators.ULT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareUGTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.UGT, m -> m.not()); - testCompareMaskNotInt(VectorOperators.UGT, m -> m.xor(I_SPECIES.maskAll(true))); + testCompareMaskNotInt(I_SPECIES, VectorOperators.UGT, (m) -> { return m.not(); }); + verifyResultsInt(I_SPECIES, VectorOperators.UGT); + testCompareMaskNotInt(I_SPECIES, VectorOperators.UGT, (m) -> { return I_SPECIES.maskAll(true).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.UGT); + + testCompareMaskNotInt(I_SPECIES_FOR_CAST, VectorOperators.UGT, (m) -> { return L_SPECIES_FOR_CAST.maskAll(true).xor(m.cast(L_SPECIES_FOR_CAST)); }); + verifyResultsInt(I_SPECIES_FOR_CAST, VectorOperators.UGT); + testCompareMaskNotInt(IntVector.SPECIES_128, VectorOperators.UGT, (m) -> { return m.cast(ShortVector.SPECIES_64).not(); }); + verifyResultsInt(IntVector.SPECIES_128, VectorOperators.UGT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareULEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.ULE, m -> m.not()); - testCompareMaskNotInt(VectorOperators.ULE, m -> m.xor(I_SPECIES.maskAll(true))); + testCompareMaskNotInt(I_SPECIES, VectorOperators.ULE, (m) -> { return m.not(); }); + verifyResultsInt(I_SPECIES, VectorOperators.ULE); + testCompareMaskNotInt(I_SPECIES, VectorOperators.ULE, (m) -> { return I_SPECIES.maskAll(true).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.ULE); + + testCompareMaskNotInt(I_SPECIES_FOR_CAST, VectorOperators.ULE, (m) -> { return L_SPECIES_FOR_CAST.maskAll(true).xor(m.cast(L_SPECIES_FOR_CAST)); }); + verifyResultsInt(I_SPECIES_FOR_CAST, VectorOperators.ULE); + testCompareMaskNotInt(IntVector.SPECIES_128, VectorOperators.ULE, (m) -> { return m.cast(ShortVector.SPECIES_64).not(); }); + verifyResultsInt(IntVector.SPECIES_128, VectorOperators.ULE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_I, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 4" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareUGEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.UGE, m -> m.not()); - testCompareMaskNotInt(VectorOperators.UGE, m -> m.xor(I_SPECIES.maskAll(true))); + testCompareMaskNotInt(I_SPECIES, VectorOperators.UGE, (m) -> { return m.not(); }); + verifyResultsInt(I_SPECIES, VectorOperators.UGE); + testCompareMaskNotInt(I_SPECIES, VectorOperators.UGE, (m) -> { return I_SPECIES.maskAll(true).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.UGE); + + testCompareMaskNotInt(I_SPECIES_FOR_CAST, VectorOperators.UGE, (m) -> { return L_SPECIES_FOR_CAST.maskAll(true).xor(m.cast(L_SPECIES_FOR_CAST)); }); + verifyResultsInt(I_SPECIES_FOR_CAST, VectorOperators.UGE); + testCompareMaskNotInt(IntVector.SPECIES_128, VectorOperators.UGE, (m) -> { return m.cast(ShortVector.SPECIES_64).not(); }); + verifyResultsInt(IntVector.SPECIES_128, VectorOperators.UGE); } // Long tests @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareEQMaskNotLong() { - testCompareMaskNotLong(VectorOperators.EQ, m -> m.not()); - testCompareMaskNotLong(VectorOperators.EQ, m -> m.xor(L_SPECIES.maskAll(true))); + testCompareMaskNotLong(L_SPECIES, VectorOperators.EQ, (m) -> { return m.not(); }); + verifyResultsLong(L_SPECIES, VectorOperators.EQ); + testCompareMaskNotLong(L_SPECIES, VectorOperators.EQ, (m) -> { return L_SPECIES.maskAll(true).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.EQ); + + testCompareMaskNotLong(L_SPECIES_FOR_CAST, VectorOperators.EQ, (m) -> { return m.cast(I_SPECIES_FOR_CAST).not(); }); + verifyResultsLong(L_SPECIES_FOR_CAST, VectorOperators.EQ); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareNEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.NE, m -> m.not()); - testCompareMaskNotLong(VectorOperators.NE, m -> m.xor(L_SPECIES.maskAll(true))); + testCompareMaskNotLong(L_SPECIES, VectorOperators.NE, (m) -> { return m.not(); }); + verifyResultsLong(L_SPECIES, VectorOperators.NE); + testCompareMaskNotLong(L_SPECIES, VectorOperators.NE, (m) -> { return L_SPECIES.maskAll(true).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.NE); + + testCompareMaskNotLong(L_SPECIES_FOR_CAST, VectorOperators.NE, (m) -> { return m.cast(I_SPECIES_FOR_CAST).not(); }); + verifyResultsLong(L_SPECIES_FOR_CAST, VectorOperators.NE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareLTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.LT, m -> m.not()); - testCompareMaskNotLong(VectorOperators.LT, m -> m.xor(L_SPECIES.maskAll(true))); + testCompareMaskNotLong(L_SPECIES, VectorOperators.LT, (m) -> { return m.not(); }); + verifyResultsLong(L_SPECIES, VectorOperators.LT); + testCompareMaskNotLong(L_SPECIES, VectorOperators.LT, (m) -> { return L_SPECIES.maskAll(true).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.LT); + + testCompareMaskNotLong(L_SPECIES_FOR_CAST, VectorOperators.LT, (m) -> { return m.cast(I_SPECIES_FOR_CAST).not(); }); + verifyResultsLong(L_SPECIES_FOR_CAST, VectorOperators.LT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareGTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.GT, m -> m.not()); - testCompareMaskNotLong(VectorOperators.GT, m -> m.xor(L_SPECIES.maskAll(true))); + testCompareMaskNotLong(L_SPECIES, VectorOperators.GT, (m) -> { return m.not(); }); + verifyResultsLong(L_SPECIES, VectorOperators.GT); + testCompareMaskNotLong(L_SPECIES, VectorOperators.GT, (m) -> { return L_SPECIES.maskAll(true).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.GT); + + testCompareMaskNotLong(L_SPECIES_FOR_CAST, VectorOperators.GT, (m) -> { return m.cast(I_SPECIES_FOR_CAST).not(); }); + verifyResultsLong(L_SPECIES_FOR_CAST, VectorOperators.GT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareLEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.LE, m -> m.not()); - testCompareMaskNotLong(VectorOperators.LE, m -> m.xor(L_SPECIES.maskAll(true))); + testCompareMaskNotLong(L_SPECIES, VectorOperators.LE, (m) -> { return m.not(); }); + verifyResultsLong(L_SPECIES, VectorOperators.LE); + testCompareMaskNotLong(L_SPECIES, VectorOperators.LE, (m) -> { return L_SPECIES.maskAll(true).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.LE); + + testCompareMaskNotLong(L_SPECIES_FOR_CAST, VectorOperators.LE, (m) -> { return m.cast(I_SPECIES_FOR_CAST).not(); }); + verifyResultsLong(L_SPECIES_FOR_CAST, VectorOperators.LE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareGEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.GE, m -> m.not()); - testCompareMaskNotLong(VectorOperators.GE, m -> m.xor(L_SPECIES.maskAll(true))); + testCompareMaskNotLong(L_SPECIES, VectorOperators.GE, (m) -> { return m.not(); }); + verifyResultsLong(L_SPECIES, VectorOperators.GE); + testCompareMaskNotLong(L_SPECIES, VectorOperators.GE, (m) -> { return L_SPECIES.maskAll(true).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.GE); + + testCompareMaskNotLong(L_SPECIES_FOR_CAST, VectorOperators.GE, (m) -> { return m.cast(I_SPECIES_FOR_CAST).not(); }); + verifyResultsLong(L_SPECIES_FOR_CAST, VectorOperators.GE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareULTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.ULT, m -> m.not()); - testCompareMaskNotLong(VectorOperators.ULT, m -> m.xor(L_SPECIES.maskAll(true))); + testCompareMaskNotLong(L_SPECIES, VectorOperators.ULT, (m) -> { return m.not(); }); + verifyResultsLong(L_SPECIES, VectorOperators.ULT); + testCompareMaskNotLong(L_SPECIES, VectorOperators.ULT, (m) -> { return L_SPECIES.maskAll(true).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.ULT); + + testCompareMaskNotLong(L_SPECIES_FOR_CAST, VectorOperators.ULT, (m) -> { return m.cast(I_SPECIES_FOR_CAST).not(); }); + verifyResultsLong(L_SPECIES_FOR_CAST, VectorOperators.ULT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareUGTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.UGT, m -> m.not()); - testCompareMaskNotLong(VectorOperators.UGT, m -> m.xor(L_SPECIES.maskAll(true))); + testCompareMaskNotLong(L_SPECIES, VectorOperators.UGT, (m) -> { return m.not(); }); + verifyResultsLong(L_SPECIES, VectorOperators.UGT); + testCompareMaskNotLong(L_SPECIES, VectorOperators.UGT, (m) -> { return L_SPECIES.maskAll(true).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.UGT); + + testCompareMaskNotLong(L_SPECIES_FOR_CAST, VectorOperators.UGT, (m) -> { return m.cast(I_SPECIES_FOR_CAST).not(); }); + verifyResultsLong(L_SPECIES_FOR_CAST, VectorOperators.UGT); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareULEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.ULE, m -> m.not()); - testCompareMaskNotLong(VectorOperators.ULE, m -> m.xor(L_SPECIES.maskAll(true))); + testCompareMaskNotLong(L_SPECIES, VectorOperators.ULE, (m) -> { return m.not(); }); + verifyResultsLong(L_SPECIES, VectorOperators.ULE); + testCompareMaskNotLong(L_SPECIES, VectorOperators.ULE, (m) -> { return L_SPECIES.maskAll(true).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.ULE); + + testCompareMaskNotLong(L_SPECIES_FOR_CAST, VectorOperators.ULE, (m) -> { return m.cast(I_SPECIES_FOR_CAST).not(); }); + verifyResultsLong(L_SPECIES_FOR_CAST, VectorOperators.ULE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_L, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareUGEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.UGE, m -> m.not()); - testCompareMaskNotLong(VectorOperators.UGE, m -> m.xor(L_SPECIES.maskAll(true))); + testCompareMaskNotLong(L_SPECIES, VectorOperators.UGE, (m) -> { return m.not(); }); + verifyResultsLong(L_SPECIES, VectorOperators.UGE); + testCompareMaskNotLong(L_SPECIES, VectorOperators.UGE, (m) -> { return L_SPECIES.maskAll(true).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.UGE); + + testCompareMaskNotLong(L_SPECIES_FOR_CAST, VectorOperators.UGE, (m) -> { return m.cast(I_SPECIES_FOR_CAST).not(); }); + verifyResultsLong(L_SPECIES_FOR_CAST, VectorOperators.UGE); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareEQMaskNotFloat() { - testCompareMaskNotFloat(VectorOperators.EQ, fa, fb, m -> m.not()); - testCompareMaskNotFloat(VectorOperators.EQ, fa, fb, m -> m.xor(F_SPECIES.maskAll(true))); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.EQ, fa, fb, (m) -> { return m.not(); }); + verifyResultsFloat(F_SPECIES, VectorOperators.EQ, fa, fb); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.EQ, fa, fb, (m) -> { return F_SPECIES.maskAll(true).xor(m); }); + verifyResultsFloat(F_SPECIES, VectorOperators.EQ, fa, fb); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VI, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareNEMaskNotFloat() { - testCompareMaskNotFloat(VectorOperators.NE, fa, fb, m -> m.not()); - testCompareMaskNotFloat(VectorOperators.NE, fa, fb, m -> m.xor(F_SPECIES.maskAll(true))); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.NE, fa, fb, (m) -> { return m.not(); }); + verifyResultsFloat(F_SPECIES, VectorOperators.NE, fa, fb); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.NE, fa, fb, (m) -> { return F_SPECIES.maskAll(true).xor(m); }); + verifyResultsFloat(F_SPECIES, VectorOperators.NE, fa, fb); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareEQMaskNotFloatNaN() { - testCompareMaskNotFloat(VectorOperators.EQ, fa, fnan, m -> m.not()); - testCompareMaskNotFloat(VectorOperators.EQ, fa, fnan, m -> m.xor(F_SPECIES.maskAll(true))); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.EQ, fa, fnan, (m) -> { return m.not(); }); + verifyResultsFloat(F_SPECIES, VectorOperators.EQ, fa, fnan); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.EQ, fa, fnan, (m) -> { return F_SPECIES.maskAll(true).xor(m); }); + verifyResultsFloat(F_SPECIES, VectorOperators.EQ, fa, fnan); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareNEMaskNotFloatNaN() { - testCompareMaskNotFloat(VectorOperators.NE, fa, fnan, m -> m.not()); - testCompareMaskNotFloat(VectorOperators.NE, fa, fnan, m -> m.xor(F_SPECIES.maskAll(true))); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.NE, fa, fnan, (m) -> { return m.not(); }); + verifyResultsFloat(F_SPECIES, VectorOperators.NE, fa, fnan); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.NE, fa, fnan, (m) -> { return F_SPECIES.maskAll(true).xor(m); }); + verifyResultsFloat(F_SPECIES, VectorOperators.NE, fa, fnan); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareEQMaskNotFloatPositiveInfinity() { - testCompareMaskNotFloat(VectorOperators.EQ, fa, fpinf, m -> m.not()); - testCompareMaskNotFloat(VectorOperators.EQ, fa, fpinf, m -> m.xor(F_SPECIES.maskAll(true))); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.EQ, fa, fpinf, (m) -> { return m.not(); }); + verifyResultsFloat(F_SPECIES, VectorOperators.EQ, fa, fpinf); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.EQ, fa, fpinf, (m) -> { return F_SPECIES.maskAll(true).xor(m); }); + verifyResultsFloat(F_SPECIES, VectorOperators.EQ, fa, fpinf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareNEMaskNotFloatPositiveInfinity() { - testCompareMaskNotFloat(VectorOperators.NE, fa, fpinf, m -> m.not()); - testCompareMaskNotFloat(VectorOperators.NE, fa, fpinf, m -> m.xor(F_SPECIES.maskAll(true))); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.NE, fa, fpinf, (m) -> { return m.not(); }); + verifyResultsFloat(F_SPECIES, VectorOperators.NE, fa, fpinf); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.NE, fa, fpinf, (m) -> { return F_SPECIES.maskAll(true).xor(m); }); + verifyResultsFloat(F_SPECIES, VectorOperators.NE, fa, fpinf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareEQMaskNotFloatNegativeInfinity() { - testCompareMaskNotFloat(VectorOperators.EQ, fa, fninf, m -> m.not()); - testCompareMaskNotFloat(VectorOperators.EQ, fa, fninf, m -> m.xor(F_SPECIES.maskAll(true))); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.EQ, fa, fninf, (m) -> { return m.not(); }); + verifyResultsFloat(F_SPECIES, VectorOperators.EQ, fa, fninf); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.EQ, fa, fninf, (m) -> { return F_SPECIES.maskAll(true).xor(m); }); + verifyResultsFloat(F_SPECIES, VectorOperators.EQ, fa, fninf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_F, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareNEMaskNotFloatNegativeInfinity() { - testCompareMaskNotFloat(VectorOperators.NE, fa, fninf, m -> m.not()); - testCompareMaskNotFloat(VectorOperators.NE, fa, fninf, m -> m.xor(F_SPECIES.maskAll(true))); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.NE, fa, fninf, (m) -> { return m.not(); }); + verifyResultsFloat(F_SPECIES, VectorOperators.NE, fa, fninf); + testCompareMaskNotFloat(F_SPECIES, VectorOperators.NE, fa, fninf, (m) -> { return F_SPECIES.maskAll(true).xor(m); }); + verifyResultsFloat(F_SPECIES, VectorOperators.NE, fa, fninf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareEQMaskNotDouble() { - testCompareMaskNotDouble(VectorOperators.EQ, da, db, m -> m.not()); - testCompareMaskNotDouble(VectorOperators.EQ, da, db, m -> m.xor(D_SPECIES.maskAll(true))); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.EQ, da, db, (m) -> { return m.not(); }); + verifyResultsDouble(D_SPECIES, VectorOperators.EQ, da, db); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.EQ, da, db, (m) -> { return D_SPECIES.maskAll(true).xor(m); }); + verifyResultsDouble(D_SPECIES, VectorOperators.EQ, da, db); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareNEMaskNotDouble() { - testCompareMaskNotDouble(VectorOperators.NE, da, db, m -> m.not()); - testCompareMaskNotDouble(VectorOperators.NE, da, db, m -> m.xor(D_SPECIES.maskAll(true))); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.NE, da, db, (m) -> { return m.not(); }); + verifyResultsDouble(D_SPECIES, VectorOperators.NE, da, db); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.NE, da, db, (m) -> { return D_SPECIES.maskAll(true).xor(m); }); + verifyResultsDouble(D_SPECIES, VectorOperators.NE, da, db); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareEQMaskNotDoubleNaN() { - testCompareMaskNotDouble(VectorOperators.EQ, da, dnan, m -> m.not()); - testCompareMaskNotDouble(VectorOperators.EQ, da, dnan, m -> m.xor(D_SPECIES.maskAll(true))); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.EQ, da, dnan, (m) -> { return m.not(); }); + verifyResultsDouble(D_SPECIES, VectorOperators.EQ, da, dnan); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.EQ, da, dnan, (m) -> { return D_SPECIES.maskAll(true).xor(m); }); + verifyResultsDouble(D_SPECIES, VectorOperators.EQ, da, dnan); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareNEMaskNotDoubleNaN() { - testCompareMaskNotDouble(VectorOperators.NE, da, dnan, m -> m.not()); - testCompareMaskNotDouble(VectorOperators.NE, da, dnan, m -> m.xor(D_SPECIES.maskAll(true))); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.NE, da, dnan, (m) -> { return m.not(); }); + verifyResultsDouble(D_SPECIES, VectorOperators.NE, da, dnan); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.NE, da, dnan, (m) -> { return D_SPECIES.maskAll(true).xor(m); }); + verifyResultsDouble(D_SPECIES, VectorOperators.NE, da, dnan); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareEQMaskNotDoublePositiveInfinity() { - testCompareMaskNotDouble(VectorOperators.EQ, da, dpinf, m -> m.not()); - testCompareMaskNotDouble(VectorOperators.EQ, da, dpinf, m -> m.xor(D_SPECIES.maskAll(true))); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.EQ, da, dpinf, (m) -> { return m.not(); }); + verifyResultsDouble(D_SPECIES, VectorOperators.EQ, da, dpinf); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.EQ, da, dpinf, (m) -> { return D_SPECIES.maskAll(true).xor(m); }); + verifyResultsDouble(D_SPECIES, VectorOperators.EQ, da, dpinf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareNEMaskNotDoublePositiveInfinity() { - testCompareMaskNotDouble(VectorOperators.NE, da, dpinf, m -> m.not()); - testCompareMaskNotDouble(VectorOperators.NE, da, dpinf, m -> m.xor(D_SPECIES.maskAll(true))); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.NE, da, dpinf, (m) -> { return m.not(); }); + verifyResultsDouble(D_SPECIES, VectorOperators.NE, da, dpinf); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.NE, da, dpinf, (m) -> { return D_SPECIES.maskAll(true).xor(m); }); + verifyResultsDouble(D_SPECIES, VectorOperators.NE, da, dpinf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareEQMaskNotDoubleNegativeInfinity() { - testCompareMaskNotDouble(VectorOperators.EQ, da, dninf, m -> m.not()); - testCompareMaskNotDouble(VectorOperators.EQ, da, dninf, m -> m.xor(D_SPECIES.maskAll(true))); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.EQ, da, dninf, (m) -> { return m.not(); }); + verifyResultsDouble(D_SPECIES, VectorOperators.EQ, da, dninf); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.EQ, da, dninf, (m) -> { return D_SPECIES.maskAll(true).xor(m); }); + verifyResultsDouble(D_SPECIES, VectorOperators.EQ, da, dninf); } @Test - @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_VL, "= 0", IRNode.VECTOR_MASK_CMP_D, "= 2" }, - applyIfCPUFeatureOr = {"asimd", "true", "avx2", "true", "rvv", "true"}) + @IR(counts = { IRNode.XOR_V_MASK, "= 0", + IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareNEMaskNotDoubleNegativeInfinity() { - testCompareMaskNotDouble(VectorOperators.NE, da, dninf, m -> m.not()); - testCompareMaskNotDouble(VectorOperators.NE, da, dninf, m -> m.xor(D_SPECIES.maskAll(true))); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.NE, da, dninf, (m) -> { return m.not(); }); + verifyResultsDouble(D_SPECIES, VectorOperators.NE, da, dninf); + testCompareMaskNotDouble(D_SPECIES, VectorOperators.NE, da, dninf, (m) -> { return D_SPECIES.maskAll(true).xor(m); }); + verifyResultsDouble(D_SPECIES, VectorOperators.NE, da, dninf); + } + + // negative tests + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "sve", "true", "avx512", "true", "rvv", "true" }) + @IR(counts = { IRNode.XOR_V, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureAnd = { "asimd", "true", "sve", "false" }) + @IR(counts = { IRNode.XOR_V, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureAnd = { "avx2", "true" }) + public static void testCompareMaskNotByteNegative() { + testCompareMaskNotByte(B_SPECIES, VectorOperators.EQ, (m) -> { + // The vector mask is used multiple times. + ic[0] = m.trueCount(); + return m.not(); + }); + verifyResultsByte(B_SPECIES, VectorOperators.EQ); + + // One of the operands of XOR is not all ones vector. + testCompareMaskNotByte(B_SPECIES, VectorOperators.EQ, (m) -> { return B_SPECIES.maskAll(false).xor(m); }); + verifyResultsByte(B_SPECIES, VectorOperators.NE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "sve", "true", "avx512", "true", "rvv", "true" }) + @IR(counts = { IRNode.XOR_V, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureAnd = { "asimd", "true", "sve", "false" }) + @IR(counts = { IRNode.XOR_V, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureAnd = { "avx2", "true" }) + public static void testCompareMaskNotShortNegative() { + testCompareMaskNotShort(S_SPECIES, VectorOperators.EQ, (m) -> { + // The vector mask is used multiple times. + ic[0] = m.trueCount(); + return m.not(); + }); + verifyResultsShort(S_SPECIES, VectorOperators.EQ); + + // One of the operands of XOR is not all ones vector. + testCompareMaskNotShort(S_SPECIES, VectorOperators.EQ, (m) -> { return S_SPECIES.maskAll(false).xor(m); }); + verifyResultsShort(S_SPECIES, VectorOperators.NE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "sve", "true", "avx512", "true", "rvv", "true" }) + @IR(counts = { IRNode.XOR_V, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureAnd = { "asimd", "true", "sve", "false" }) + @IR(counts = { IRNode.XOR_V, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureAnd = { "avx2", "true" }) + public static void testCompareMaskNotIntNegative() { + testCompareMaskNotInt(I_SPECIES, VectorOperators.EQ, (m) -> { + // The vector mask is used multiple times. + ic[0] = m.trueCount(); + return m.not(); + }); + verifyResultsInt(I_SPECIES, VectorOperators.EQ); + + // One of the operands of XOR is not all ones vector. + testCompareMaskNotInt(I_SPECIES, VectorOperators.EQ, (m) -> { return I_SPECIES.maskAll(false).xor(m); }); + verifyResultsInt(I_SPECIES, VectorOperators.NE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureOr = { "sve", "true", "avx512", "true", "rvv", "true" }) + @IR(counts = { IRNode.XOR_V, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureAnd = { "asimd", "true", "sve", "false" }) + @IR(counts = { IRNode.XOR_V, "= 2", + IRNode.VECTOR_MASK_CMP, "= 2" }, + applyIfCPUFeatureAnd = { "avx2", "true" }) + public static void testCompareMaskNotLongNegative() { + testCompareMaskNotLong(L_SPECIES, VectorOperators.EQ, (m) -> { + // The vector mask is used multiple times. + ic[0] = m.trueCount(); + return m.not(); + }); + verifyResultsLong(L_SPECIES, VectorOperators.EQ); + + // One of the operands of XOR is not all ones vector. + testCompareMaskNotLong(L_SPECIES, VectorOperators.EQ, (m) -> { return L_SPECIES.maskAll(false).xor(m); }); + verifyResultsLong(L_SPECIES, VectorOperators.NE); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 3", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "sve", "true", "avx512", "true", "rvv", "true" }) + @IR(counts = { IRNode.XOR_V, "= 3", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureAnd = { "asimd", "true", "sve", "false" }) + @IR(counts = { IRNode.XOR_V, "= 3", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureAnd = { "avx2", "true" }) + public static void testCompareMaskNotFloatNegative() { + testCompareMaskNotFloat(F_SPECIES, VectorOperators.EQ, fa, fb, (m) -> { + // The vector mask is used multiple times. + ic[0] = m.trueCount(); + return m.not(); + }); + verifyResultsFloat(F_SPECIES, VectorOperators.EQ, fa, fb); + + // One of the operands of XOR is not all ones vector. + testCompareMaskNotFloat(F_SPECIES, VectorOperators.EQ, fa, fb, (m) -> { return F_SPECIES.maskAll(false).xor(m); }); + verifyResultsFloat(F_SPECIES, VectorOperators.NE, fa, fb); + + // Float vectors use the LT comparison. + testCompareMaskNotFloat(F_SPECIES, VectorOperators.LT, fa, fb, (m) -> { return m.not(); }); + verifyResultsFloat(F_SPECIES, VectorOperators.LT, fa, fb); + } + + @Test + @IR(counts = { IRNode.XOR_V_MASK, "= 3", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureOr = { "sve", "true", "avx512", "true", "rvv", "true" }) + @IR(counts = { IRNode.XOR_V, "= 3", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureAnd = { "asimd", "true", "sve", "false" }) + @IR(counts = { IRNode.XOR_V, "= 3", + IRNode.VECTOR_MASK_CMP, "= 3" }, + applyIfCPUFeatureAnd = { "avx2", "true" }) + public static void testCompareMaskNotDoubleNegative() { + testCompareMaskNotDouble(D_SPECIES, VectorOperators.EQ, da, db, (m) -> { + // The vector mask is used multiple times. + ic[0] = m.trueCount(); + return m.not(); + }); + verifyResultsDouble(D_SPECIES, VectorOperators.EQ, da, db); + + // One of the operands of XOR is not all ones vector. + testCompareMaskNotDouble(D_SPECIES, VectorOperators.EQ, da, db, (m) -> { return D_SPECIES.maskAll(false).xor(m); }); + verifyResultsDouble(D_SPECIES, VectorOperators.NE, da, db); + + // Double vectors use the LT comparison. + testCompareMaskNotDouble(D_SPECIES, VectorOperators.LT, da, db, (m) -> { return m.not(); }); + verifyResultsDouble(D_SPECIES, VectorOperators.LT, da, db); } public static void main(String[] args) { TestFramework testFramework = new TestFramework(); - testFramework.addFlags("--add-modules=jdk.incubator.vector"); - testFramework.setDefaultWarmup(10000); - testFramework.start(); + testFramework.setDefaultWarmup(5000) + .addFlags("--add-modules=jdk.incubator.vector") + .start(); } } From db78dc438610ed5bcfbf7ae1191c02b10bf58391 Mon Sep 17 00:00:00 2001 From: erfang Date: Tue, 8 Jul 2025 05:51:49 +0000 Subject: [PATCH 09/12] Align indentation --- src/hotspot/share/opto/vectornode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 83dc31198d834..82e1a82add4a0 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2238,7 +2238,7 @@ Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { if (in1->Opcode() != Op_VectorMaskCmp || in1->outcnt() != 1 || !(in1->as_VectorMaskCmp())->predicate_can_be_negated() || - !VectorNode::is_all_ones_vector(in2)) { + !VectorNode::is_all_ones_vector(in2)) { return nullptr; } From 04142a19ee9ba8ab986e3aa9344e7c2e95eca82f Mon Sep 17 00:00:00 2001 From: erfang Date: Wed, 9 Jul 2025 06:04:25 +0000 Subject: [PATCH 10/12] Update the code comment --- src/hotspot/share/opto/vectornode.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 82e1a82add4a0..8bb8ef77c0ce8 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2226,8 +2226,11 @@ Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { } bool with_vector_mask_cast = false; - // VectorMaskCast and VectorMaskCmp should only have a single use, - // otherwise the optimization may be unprofitable. + // Required conditions: + // 1. VectorMaskCast and VectorMaskCmp should only have a single use, + // otherwise the optimization may be unprofitable. + // 2. The predicate of VectorMaskCmp should be negatable. + // 3. The second input should be an all true vector mask. if (in1->Opcode() == Op_VectorMaskCast) { if (in1->outcnt() != 1) { return nullptr; @@ -2238,7 +2241,7 @@ Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { if (in1->Opcode() != Op_VectorMaskCmp || in1->outcnt() != 1 || !(in1->as_VectorMaskCmp())->predicate_can_be_negated() || - !VectorNode::is_all_ones_vector(in2)) { + !VectorNode::is_all_ones_vector(in2)) { return nullptr; } From 52bbd3cd863219d3b26fad769eebe5ce6e72657c Mon Sep 17 00:00:00 2001 From: erfang Date: Mon, 15 Sep 2025 02:16:50 +0000 Subject: [PATCH 11/12] Simplify JMH testing --- src/hotspot/share/opto/vectornode.cpp | 8 +- .../vector/MaskCompareNotBenchmark.java | 407 ++++++------------ 2 files changed, 129 insertions(+), 286 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index b785d34f73b39..ae9ef552df4b5 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2345,7 +2345,7 @@ Node* XorVNode::Ideal_XorV_VectorMaskCmp(PhaseGVN* phase, bool can_reshape) { } if (in1->Opcode() != Op_VectorMaskCmp || in1->outcnt() != 1 || - !(in1->as_VectorMaskCmp())->predicate_can_be_negated() || + !in1->as_VectorMaskCmp()->predicate_can_be_negated() || !VectorNode::is_all_ones_vector(in2)) { return nullptr; } @@ -2376,10 +2376,10 @@ Node* XorVNode::Ideal(PhaseGVN* phase, bool can_reshape) { } Node* res = Ideal_XorV_VectorMaskCmp(phase, can_reshape); - if (res == nullptr) { - res = VectorNode::Ideal(phase, can_reshape); + if (res != nullptr) { + return res; } - return res; + return VectorNode::Ideal(phase, can_reshape); } Node* VectorBlendNode::Identity(PhaseGVN* phase) { diff --git a/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java b/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java index 9e60fc81cfcd2..d83bc126a1d06 100644 --- a/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java +++ b/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskCompareNotBenchmark.java @@ -27,6 +27,7 @@ import org.openjdk.jmh.infra.*; import jdk.incubator.vector.*; +import java.lang.invoke.*; import java.util.concurrent.TimeUnit; import java.util.Random; @@ -35,35 +36,43 @@ @State(Scope.Thread) @Warmup(iterations = 5, time = 1) @Measurement(iterations = 5, time = 1) -@Fork(jvmArgs = { "--add-modules=jdk.incubator.vector" }) -public class MaskCompareNotBenchmark { +@Fork(value = 2, jvmArgs = { "--add-modules=jdk.incubator.vector" }) +public abstract class MaskCompareNotBenchmark { @Param({"4096"}) - private int ARRAYLEN; - private static Random r = new Random(); + protected int ARRAYLEN; + + // Abstract method to get comparison operator from subclasses + protected abstract String getComparisonOperatorName(); - private static final VectorSpecies B_SPECIES = ByteVector.SPECIES_MAX; - private static final VectorSpecies S_SPECIES = ShortVector.SPECIES_MAX; - private static final VectorSpecies I_SPECIES = IntVector.SPECIES_MAX; - private static final VectorSpecies L_SPECIES = LongVector.SPECIES_MAX; - private static final VectorSpecies F_SPECIES = FloatVector.SPECIES_MAX; - private static final VectorSpecies D_SPECIES = DoubleVector.SPECIES_MAX; + // To get compile-time constants for comparison operation + static final MutableCallSite MUTABLE_COMPARISON_CONSTANT = new MutableCallSite(MethodType.methodType(VectorOperators.Comparison.class)); + static final MethodHandle MUTABLE_COMPARISON_CONSTANT_HANDLE = MUTABLE_COMPARISON_CONSTANT.dynamicInvoker(); + + private static Random r = new Random(); - boolean[] mr; - byte[] ba; - byte[] bb; - short[] sa; - short[] sb; - int[] ia; - int[] ib; - long[] la; - long[] lb; - float[] fa; - float[] fb; - double[] da; - double[] db; + protected static final VectorSpecies B_SPECIES = ByteVector.SPECIES_MAX; + protected static final VectorSpecies S_SPECIES = ShortVector.SPECIES_MAX; + protected static final VectorSpecies I_SPECIES = IntVector.SPECIES_MAX; + protected static final VectorSpecies L_SPECIES = LongVector.SPECIES_MAX; + protected static final VectorSpecies F_SPECIES = FloatVector.SPECIES_MAX; + protected static final VectorSpecies D_SPECIES = DoubleVector.SPECIES_MAX; + + protected boolean[] mr; + protected byte[] ba; + protected byte[] bb; + protected short[] sa; + protected short[] sb; + protected int[] ia; + protected int[] ib; + protected long[] la; + protected long[] lb; + protected float[] fa; + protected float[] fb; + protected double[] da; + protected double[] db; @Setup - public void init() { + public void init() throws Throwable { mr = new boolean[ARRAYLEN]; ba = new byte[ARRAYLEN]; bb = new byte[ARRAYLEN]; @@ -93,285 +102,119 @@ public void init() { da[i] = r.nextDouble(); db[i] = r.nextDouble(); } - } - @CompilerControl(CompilerControl.Mode.INLINE) - private void testCompareMaskNotByte(VectorOperators.Comparison op) { - ByteVector bv = ByteVector.fromArray(B_SPECIES, bb, 0); - for (int j = 0; j < ARRAYLEN; j += B_SPECIES.length()) { - ByteVector av = ByteVector.fromArray(B_SPECIES, ba, j); - VectorMask m = av.compare(op, bv).not(); - m.intoArray(mr, j); - } + VectorOperators.Comparison comparisonOp = getComparisonOperator(getComparisonOperatorName()); + MethodHandle constant = MethodHandles.constant(VectorOperators.Comparison.class, comparisonOp); + MUTABLE_COMPARISON_CONSTANT.setTarget(constant); } @CompilerControl(CompilerControl.Mode.INLINE) - private void testCompareMaskNotShort(VectorOperators.Comparison op) { - ShortVector bv = ShortVector.fromArray(S_SPECIES, sb, 0); - for (int j = 0; j < ARRAYLEN; j += S_SPECIES.length()) { - ShortVector av = ShortVector.fromArray(S_SPECIES, sa, j); - VectorMask m = av.compare(op, bv).not(); - m.intoArray(mr, j); + private static VectorOperators.Comparison getComparisonOperator(String op) { + switch (op) { + case "EQ": return VectorOperators.EQ; + case "NE": return VectorOperators.NE; + case "LT": return VectorOperators.LT; + case "LE": return VectorOperators.LE; + case "GT": return VectorOperators.GT; + case "GE": return VectorOperators.GE; + case "ULT": return VectorOperators.ULT; + case "ULE": return VectorOperators.ULE; + case "UGT": return VectorOperators.UGT; + case "UGE": return VectorOperators.UGE; + default: throw new IllegalArgumentException("Unknown comparison operator: " + op); } } @CompilerControl(CompilerControl.Mode.INLINE) - private void testCompareMaskNotInt(VectorOperators.Comparison op) { - IntVector bv = IntVector.fromArray(I_SPECIES, ib, 0); - for (int j = 0; j < ARRAYLEN; j += I_SPECIES.length()) { - IntVector av = IntVector.fromArray(I_SPECIES, ia, j); - VectorMask m = av.compare(op, bv).not(); - m.intoArray(mr, j); - } + protected VectorOperators.Comparison comparison_con() throws Throwable { + return (VectorOperators.Comparison) MUTABLE_COMPARISON_CONSTANT_HANDLE.invokeExact(); } - @CompilerControl(CompilerControl.Mode.INLINE) - private void testCompareMaskNotLong(VectorOperators.Comparison op) { - LongVector bv = LongVector.fromArray(L_SPECIES, lb, 0); - for (int j = 0; j < ARRAYLEN; j += L_SPECIES.length()) { - LongVector av = LongVector.fromArray(L_SPECIES, la, j); - VectorMask m = av.compare(op, bv).not(); - m.intoArray(mr, j); - } - } + // Subclasses with different comparison operators + public static class IntegerComparisons extends MaskCompareNotBenchmark { + @Param({"EQ", "NE", "LT", "LE", "GT", "GE", "ULT", "ULE", "UGT", "UGE"}) + public String COMPARISON_OP; - @CompilerControl(CompilerControl.Mode.INLINE) - private void testCompareMaskNotFloat(VectorOperators.Comparison op) { - FloatVector bv = FloatVector.fromArray(F_SPECIES, fb, 0); - for (int j = 0; j < ARRAYLEN; j += F_SPECIES.length()) { - FloatVector av = FloatVector.fromArray(F_SPECIES, fa, j); - VectorMask m = av.compare(op, bv).not(); - m.intoArray(mr, j); + @Override + protected String getComparisonOperatorName() { + return COMPARISON_OP; } - } - @CompilerControl(CompilerControl.Mode.INLINE) - private void testCompareMaskNotDouble(VectorOperators.Comparison op) { - DoubleVector bv = DoubleVector.fromArray(D_SPECIES, db, 0); - for (int j = 0; j < ARRAYLEN; j += D_SPECIES.length()) { - DoubleVector av = DoubleVector.fromArray(D_SPECIES, da, j); - VectorMask m = av.compare(op, bv).not(); - m.intoArray(mr, j); + @Benchmark + public void testCompareMaskNotByte() throws Throwable { + VectorOperators.Comparison op = comparison_con(); + ByteVector bv = ByteVector.fromArray(B_SPECIES, bb, 0); + for (int j = 0; j < ARRAYLEN; j += B_SPECIES.length()) { + ByteVector av = ByteVector.fromArray(B_SPECIES, ba, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } } - } - - @Benchmark - public void testCompareEQMaskNotByte() { - testCompareMaskNotByte(VectorOperators.EQ); - } - - @Benchmark - public void testCompareNEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.NE); - } - - @Benchmark - public void testCompareLTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.LT); - } - - @Benchmark - public void testCompareGTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.GT); - } - - @Benchmark - public void testCompareLEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.LE); - } - - @Benchmark - public void testCompareGEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.GE); - } - - @Benchmark - public void testCompareULTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.ULT); - } - - @Benchmark - public void testCompareUGTMaskNotByte() { - testCompareMaskNotByte(VectorOperators.UGT); - } - - @Benchmark - public void testCompareULEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.ULE); - } - - @Benchmark - public void testCompareUGEMaskNotByte() { - testCompareMaskNotByte(VectorOperators.UGE); - } - - @Benchmark - public void testCompareEQMaskNotShort() { - testCompareMaskNotShort(VectorOperators.EQ); - } - - @Benchmark - public void testCompareNEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.NE); - } - - @Benchmark - public void testCompareLTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.LT); - } - - @Benchmark - public void testCompareGTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.GT); - } - - @Benchmark - public void testCompareLEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.LE); - } - - @Benchmark - public void testCompareGEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.GE); - } - - @Benchmark - public void testCompareULTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.ULT); - } - - @Benchmark - public void testCompareUGTMaskNotShort() { - testCompareMaskNotShort(VectorOperators.UGT); - } - - @Benchmark - public void testCompareULEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.ULE); - } - - @Benchmark - public void testCompareUGEMaskNotShort() { - testCompareMaskNotShort(VectorOperators.UGE); - } - - @Benchmark - public void testCompareEQMaskNotInt() { - testCompareMaskNotInt(VectorOperators.EQ); - } - - @Benchmark - public void testCompareNEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.NE); - } - - @Benchmark - public void testCompareLTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.LT); - } - - @Benchmark - public void testCompareGTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.GT); - } - - @Benchmark - public void testCompareLEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.LE); - } - - @Benchmark - public void testCompareGEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.GE); - } - - @Benchmark - public void testCompareULTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.ULT); - } - - @Benchmark - public void testCompareUGTMaskNotInt() { - testCompareMaskNotInt(VectorOperators.UGT); - } - - @Benchmark - public void testCompareULEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.ULE); - } - - @Benchmark - public void testCompareUGEMaskNotInt() { - testCompareMaskNotInt(VectorOperators.UGE); - } - @Benchmark - public void testCompareEQMaskNotLong() { - testCompareMaskNotLong(VectorOperators.EQ); - } - - @Benchmark - public void testCompareNEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.NE); - } - - @Benchmark - public void testCompareLTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.LT); - } - - @Benchmark - public void testCompareGTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.GT); - } - - @Benchmark - public void testCompareLEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.LE); - } - - @Benchmark - public void testCompareGEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.GE); - } - - @Benchmark - public void testCompareULTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.ULT); - } - - @Benchmark - public void testCompareUGTMaskNotLong() { - testCompareMaskNotLong(VectorOperators.UGT); - } + @Benchmark + public void testCompareMaskNotShort() throws Throwable { + VectorOperators.Comparison op = comparison_con(); + ShortVector bv = ShortVector.fromArray(S_SPECIES, sb, 0); + for (int j = 0; j < ARRAYLEN; j += S_SPECIES.length()) { + ShortVector av = ShortVector.fromArray(S_SPECIES, sa, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } - @Benchmark - public void testCompareULEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.ULE); - } + @Benchmark + public void testCompareMaskNotInt() throws Throwable { + VectorOperators.Comparison op = comparison_con(); + IntVector bv = IntVector.fromArray(I_SPECIES, ib, 0); + for (int j = 0; j < ARRAYLEN; j += I_SPECIES.length()) { + IntVector av = IntVector.fromArray(I_SPECIES, ia, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } - @Benchmark - public void testCompareUGEMaskNotLong() { - testCompareMaskNotLong(VectorOperators.UGE); + @Benchmark + public void testCompareMaskNotLong() throws Throwable { + VectorOperators.Comparison op = comparison_con(); + LongVector bv = LongVector.fromArray(L_SPECIES, lb, 0); + for (int j = 0; j < ARRAYLEN; j += L_SPECIES.length()) { + LongVector av = LongVector.fromArray(L_SPECIES, la, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } } - @Benchmark - public void testCompareEQMaskNotFloat() { - testCompareMaskNotFloat(VectorOperators.EQ); - } + public static class FloatingPointComparisons extends MaskCompareNotBenchmark { + // "ULT", "ULE", "UGT", "UGE" are not supported for floating point types + @Param({"EQ", "NE", "LT", "LE", "GT", "GE"}) + public String COMPARISON_OP; - @Benchmark - public void testCompareNEMaskNotFloat() { - testCompareMaskNotFloat(VectorOperators.NE); - } + @Override + protected String getComparisonOperatorName() { + return COMPARISON_OP; + } - @Benchmark - public void testCompareEQMaskNotDouble() { - testCompareMaskNotDouble(VectorOperators.EQ); - } + @Benchmark + public void testCompareMaskNotFloat() throws Throwable { + VectorOperators.Comparison op = comparison_con(); + FloatVector bv = FloatVector.fromArray(F_SPECIES, fb, 0); + for (int j = 0; j < ARRAYLEN; j += F_SPECIES.length()) { + FloatVector av = FloatVector.fromArray(F_SPECIES, fa, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } - @Benchmark - public void testCompareNEMaskNotDouble() { - testCompareMaskNotDouble(VectorOperators.NE); + @Benchmark + public void testCompareMaskNotDouble() throws Throwable { + VectorOperators.Comparison op = comparison_con(); + DoubleVector bv = DoubleVector.fromArray(D_SPECIES, db, 0); + for (int j = 0; j < ARRAYLEN; j += D_SPECIES.length()) { + DoubleVector av = DoubleVector.fromArray(D_SPECIES, da, j); + VectorMask m = av.compare(op, bv).not(); + m.intoArray(mr, j); + } + } } } From 56bb34ffe3ca104c8f838a41f33b1d90bb10b68b Mon Sep 17 00:00:00 2001 From: erfang Date: Mon, 15 Sep 2025 05:36:30 +0000 Subject: [PATCH 12/12] Add an IR rule for vector mask cast operation --- .../vectorapi/VectorMaskCompareNotTest.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java index 1c3dd542ef41a..851113ea4dee5 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java @@ -270,6 +270,7 @@ public static void testCompareMaskNotDouble(VectorSpecies vs, VectorOper @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareEQMaskNotByte() { @@ -285,6 +286,7 @@ public static void testCompareEQMaskNotByte() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareNEMaskNotByte() { @@ -300,6 +302,7 @@ public static void testCompareNEMaskNotByte() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareLTMaskNotByte() { @@ -315,6 +318,7 @@ public static void testCompareLTMaskNotByte() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareGTMaskNotByte() { @@ -330,6 +334,7 @@ public static void testCompareGTMaskNotByte() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareLEMaskNotByte() { @@ -345,6 +350,7 @@ public static void testCompareLEMaskNotByte() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareGEMaskNotByte() { @@ -360,6 +366,7 @@ public static void testCompareGEMaskNotByte() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareULTMaskNotByte() { @@ -375,6 +382,7 @@ public static void testCompareULTMaskNotByte() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareUGTMaskNotByte() { @@ -390,6 +398,7 @@ public static void testCompareUGTMaskNotByte() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareULEMaskNotByte() { @@ -405,6 +414,7 @@ public static void testCompareULEMaskNotByte() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareUGEMaskNotByte() { @@ -421,6 +431,7 @@ public static void testCompareUGEMaskNotByte() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareEQMaskNotShort() { @@ -438,6 +449,7 @@ public static void testCompareEQMaskNotShort() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareNEMaskNotShort() { @@ -455,6 +467,7 @@ public static void testCompareNEMaskNotShort() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareLTMaskNotShort() { @@ -472,6 +485,7 @@ public static void testCompareLTMaskNotShort() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareGTMaskNotShort() { @@ -489,6 +503,7 @@ public static void testCompareGTMaskNotShort() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareLEMaskNotShort() { @@ -506,6 +521,7 @@ public static void testCompareLEMaskNotShort() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareGEMaskNotShort() { @@ -523,6 +539,7 @@ public static void testCompareGEMaskNotShort() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareULTMaskNotShort() { @@ -540,6 +557,7 @@ public static void testCompareULTMaskNotShort() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareUGTMaskNotShort() { @@ -557,6 +575,7 @@ public static void testCompareUGTMaskNotShort() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareULEMaskNotShort() { @@ -574,6 +593,7 @@ public static void testCompareULEMaskNotShort() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx", "true", "rvv", "true" }) public static void testCompareUGEMaskNotShort() { @@ -592,6 +612,7 @@ public static void testCompareUGEMaskNotShort() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareEQMaskNotInt() { @@ -609,6 +630,7 @@ public static void testCompareEQMaskNotInt() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareNEMaskNotInt() { @@ -626,6 +648,7 @@ public static void testCompareNEMaskNotInt() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareLTMaskNotInt() { @@ -643,6 +666,7 @@ public static void testCompareLTMaskNotInt() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareGTMaskNotInt() { @@ -660,6 +684,7 @@ public static void testCompareGTMaskNotInt() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareLEMaskNotInt() { @@ -677,6 +702,7 @@ public static void testCompareLEMaskNotInt() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareGEMaskNotInt() { @@ -694,6 +720,7 @@ public static void testCompareGEMaskNotInt() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareULTMaskNotInt() { @@ -711,6 +738,7 @@ public static void testCompareULTMaskNotInt() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareUGTMaskNotInt() { @@ -728,6 +756,7 @@ public static void testCompareUGTMaskNotInt() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareULEMaskNotInt() { @@ -745,6 +774,7 @@ public static void testCompareULEMaskNotInt() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 2", IRNode.VECTOR_MASK_CMP, "= 4" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareUGEMaskNotInt() { @@ -763,6 +793,7 @@ public static void testCompareUGEMaskNotInt() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareEQMaskNotLong() { @@ -778,6 +809,7 @@ public static void testCompareEQMaskNotLong() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareNEMaskNotLong() { @@ -793,6 +825,7 @@ public static void testCompareNEMaskNotLong() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareLTMaskNotLong() { @@ -808,6 +841,7 @@ public static void testCompareLTMaskNotLong() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareGTMaskNotLong() { @@ -823,6 +857,7 @@ public static void testCompareGTMaskNotLong() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareLEMaskNotLong() { @@ -838,6 +873,7 @@ public static void testCompareLEMaskNotLong() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareGEMaskNotLong() { @@ -853,6 +889,7 @@ public static void testCompareGEMaskNotLong() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareULTMaskNotLong() { @@ -868,6 +905,7 @@ public static void testCompareULTMaskNotLong() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareUGTMaskNotLong() { @@ -883,6 +921,7 @@ public static void testCompareUGTMaskNotLong() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareULEMaskNotLong() { @@ -898,6 +937,7 @@ public static void testCompareULEMaskNotLong() { @Test @IR(counts = { IRNode.XOR_V_MASK, "= 0", IRNode.XOR_V, "= 0", + IRNode.VECTOR_MASK_CAST, "= 1", IRNode.VECTOR_MASK_CMP, "= 3" }, applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }) public static void testCompareUGEMaskNotLong() {