diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 04b1f8429d93c..208fa8475fd4f 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -6140,6 +6140,7 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case TargetOpcode::G_SEXT_INREG:
   case TargetOpcode::G_ABS:
   case TargetOpcode::G_CTLZ:
+  case TargetOpcode::G_CTPOP:
     if (TypeIdx != 0)
       return UnableToLegalize;
     Observer.changingInstr(MI);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index d8245ce2a57ba..c36b20badfc09 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -323,7 +323,13 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .clampScalar(0, s32, s128)
       .widenScalarToNextPow2(0)
       .minScalarEltSameAsIf(always, 1, 0)
-      .maxScalarEltSameAsIf(always, 1, 0);
+      .maxScalarEltSameAsIf(always, 1, 0)
+      .clampNumElements(0, v8s8, v16s8)
+      .clampNumElements(0, v4s16, v8s16)
+      .clampNumElements(0, v2s32, v4s32)
+      .clampNumElements(0, v2s64, v2s64)
+      .moreElementsToNextPow2(0)
+      .scalarizeIf(scalarOrEltWiderThan(0, 64), 0);
 
   getActionDefinitionsBuilder(G_CTLZ)
       .legalFor({{s32, s32},
diff --git a/llvm/test/CodeGen/AArch64/ctpop.ll b/llvm/test/CodeGen/AArch64/ctpop.ll
index 2299b5c5a5af9..c7c378d3e67cd 100644
--- a/llvm/test/CodeGen/AArch64/ctpop.ll
+++ b/llvm/test/CodeGen/AArch64/ctpop.ll
@@ -1,37 +1,35 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
-; CHECK-GI:       warning: Instruction selection used fallback path for v2i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v32i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v2i16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v16i16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v8i32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v2i128
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i128
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i128
 
 define void @v2i8(ptr %p1) {
-; CHECK-LABEL: v2i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    ldrb w9, [x0, #1]
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v0.s[1], w9
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    strb w9, [x0]
-; CHECK-NEXT:    strb w8, [x0, #1]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v2i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    ldrb w9, [x0, #1]
+; CHECK-SD-NEXT:    fmov s0, w8
+; CHECK-SD-NEXT:    mov v0.s[1], w9
+; CHECK-SD-NEXT:    cnt v0.8b, v0.8b
+; CHECK-SD-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-SD-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    strb w9, [x0]
+; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr b0, [x0]
+; CHECK-GI-NEXT:    ldr b1, [x0, #1]
+; CHECK-GI-NEXT:    add x8, x0, #1
+; CHECK-GI-NEXT:    mov v0.b[0], v0.b[0]
+; CHECK-GI-NEXT:    mov v0.b[1], v1.b[0]
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    st1 { v0.b }[0], [x0]
+; CHECK-GI-NEXT:    st1 { v0.b }[1], [x8]
+; CHECK-GI-NEXT:    ret
 entry:
   %d = load <2 x i8>, ptr %p1
   %s = call <2 x i8> @llvm.ctpop(<2 x i8> %d)
@@ -40,23 +38,39 @@ entry:
 }
 
 define void @v3i8(ptr %p1) {
-; CHECK-LABEL: v3i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    zip1 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-NEXT:    umov w8, v0.h[2]
-; CHECK-NEXT:    str s1, [sp, #12]
-; CHECK-NEXT:    ldrh w9, [sp, #12]
-; CHECK-NEXT:    strb w8, [x0, #2]
-; CHECK-NEXT:    strh w9, [x0]
-; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v3i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sub sp, sp, #16
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    zip1 v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    cnt v0.8b, v0.8b
+; CHECK-SD-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    str s1, [sp, #12]
+; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
+; CHECK-SD-NEXT:    strb w8, [x0, #2]
+; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    add sp, sp, #16
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr b0, [x0]
+; CHECK-GI-NEXT:    ldr b1, [x0, #1]
+; CHECK-GI-NEXT:    add x8, x0, #1
+; CHECK-GI-NEXT:    add x9, x0, #2
+; CHECK-GI-NEXT:    mov v0.b[0], v0.b[0]
+; CHECK-GI-NEXT:    mov v0.b[1], v1.b[0]
+; CHECK-GI-NEXT:    ldr b1, [x0, #2]
+; CHECK-GI-NEXT:    mov v0.b[2], v1.b[0]
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    st1 { v0.b }[0], [x0]
+; CHECK-GI-NEXT:    st1 { v0.b }[1], [x8]
+; CHECK-GI-NEXT:    st1 { v0.b }[2], [x9]
+; CHECK-GI-NEXT:    ret
 entry:
   %d = load <3 x i8>, ptr %p1
   %s = call <3 x i8> @llvm.ctpop(<3 x i8> %d)
@@ -65,15 +79,31 @@ entry:
 }
 
 define void @v4i8(ptr %p1) {
-; CHECK-LABEL: v4i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT:    str s0, [x0]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    cnt v0.8b, v0.8b
+; CHECK-SD-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-SD-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov v2.b[0], v0.b[0]
+; CHECK-GI-NEXT:    mov b3, v0.b[2]
+; CHECK-GI-NEXT:    mov b0, v0.b[3]
+; CHECK-GI-NEXT:    mov v2.b[1], v1.b[0]
+; CHECK-GI-NEXT:    mov v2.b[2], v3.b[0]
+; CHECK-GI-NEXT:    mov v2.b[3], v0.b[0]
+; CHECK-GI-NEXT:    cnt v0.8b, v2.8b
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    str w8, [x0]
+; CHECK-GI-NEXT:    ret
 entry:
   %d = load <4 x i8>, ptr %p1
   %s = call <4 x i8> @llvm.ctpop(<4 x i8> %d)
@@ -113,19 +143,30 @@ entry:
 }
 
 define void @v2i16(ptr %p1) {
-; CHECK-LABEL: v2i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    ldrh w9, [x0, #2]
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v0.s[1], w9
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    str h0, [x0]
-; CHECK-NEXT:    str h1, [x0, #2]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrh w8, [x0]
+; CHECK-SD-NEXT:    ldrh w9, [x0, #2]
+; CHECK-SD-NEXT:    fmov s0, w8
+; CHECK-SD-NEXT:    mov v0.s[1], w9
+; CHECK-SD-NEXT:    cnt v0.8b, v0.8b
+; CHECK-SD-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-SD-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str h0, [x0]
+; CHECK-SD-NEXT:    str h1, [x0, #2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    add x8, x0, #2
+; CHECK-GI-NEXT:    ld1 { v0.h }[1], [x8]
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-GI-NEXT:    str h0, [x0]
+; CHECK-GI-NEXT:    st1 { v0.h }[1], [x8]
+; CHECK-GI-NEXT:    ret
 entry:
   %d = load <2 x i16>, ptr %p1
   %s = call <2 x i16> @llvm.ctpop(<2 x i16> %d)
@@ -134,15 +175,29 @@ entry:
 }
 
 define void @v3i16(ptr %p1) {
-; CHECK-LABEL: v3i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    add x8, x0, #4
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    st1 { v0.h }[2], [x8]
-; CHECK-NEXT:    str s0, [x0]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v3i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    add x8, x0, #4
+; CHECK-SD-NEXT:    cnt v0.8b, v0.8b
+; CHECK-SD-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr h0, [x0]
+; CHECK-GI-NEXT:    add x8, x0, #2
+; CHECK-GI-NEXT:    add x9, x0, #4
+; CHECK-GI-NEXT:    ld1 { v0.h }[1], [x8]
+; CHECK-GI-NEXT:    ld1 { v0.h }[2], [x9]
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-GI-NEXT:    str h0, [x0]
+; CHECK-GI-NEXT:    st1 { v0.h }[1], [x8]
+; CHECK-GI-NEXT:    st1 { v0.h }[2], [x9]
+; CHECK-GI-NEXT:    ret
 entry:
   %d = load <3 x i16>, ptr %p1
   %s = call <3 x i16> @llvm.ctpop(<3 x i16> %d)
@@ -250,25 +305,42 @@ entry:
 }
 
 define <3 x i64> @v3i64(<3 x i64> %d) {
-; CHECK-LABEL: v3i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    cnt v1.16b, v2.16b
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v1.8h, v1.16b
-; CHECK-NEXT:    uaddlp v2.4s, v1.8h
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v2.2d, v2.4s
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-NEXT:    uaddlp v0.2d, v0.4s
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    cnt v1.16b, v2.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-SD-NEXT:    uaddlp v2.4s, v1.8h
+; CHECK-SD-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-SD-NEXT:    uaddlp v2.2d, v2.4s
+; CHECK-SD-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    cnt v1.8b, v2.8b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    uaddlv h1, v1.8b
+; CHECK-GI-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-GI-NEXT:    mov w8, v1.s[0]
+; CHECK-GI-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <3 x i64> @llvm.ctpop(<3 x i64> %d)
   ret <3 x i64> %s
@@ -292,85 +364,148 @@ entry:
 }
 
 define <2 x i128> @v2i128(<2 x i128> %d) {
-; CHECK-LABEL: v2i128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov d0, x2
-; CHECK-NEXT:    fmov d1, x0
-; CHECK-NEXT:    mov v1.d[1], x1
-; CHECK-NEXT:    mov v0.d[1], x3
-; CHECK-NEXT:    mov x1, xzr
-; CHECK-NEXT:    mov x3, xzr
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    addv b1, v1.16b
-; CHECK-NEXT:    addv b0, v0.16b
-; CHECK-NEXT:    fmov x0, d1
-; CHECK-NEXT:    fmov x2, d0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v2i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov d0, x2
+; CHECK-SD-NEXT:    fmov d1, x0
+; CHECK-SD-NEXT:    mov v1.d[1], x1
+; CHECK-SD-NEXT:    mov v0.d[1], x3
+; CHECK-SD-NEXT:    mov x1, xzr
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    cnt v1.16b, v1.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    addv b1, v1.16b
+; CHECK-SD-NEXT:    addv b0, v0.16b
+; CHECK-SD-NEXT:    fmov x0, d1
+; CHECK-SD-NEXT:    fmov x2, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v0.d[0], x0
+; CHECK-GI-NEXT:    mov v1.d[0], x2
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    mov v1.d[1], x3
+; CHECK-GI-NEXT:    mov x1, xzr
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v1.16b, v1.16b
+; CHECK-GI-NEXT:    uaddlv h0, v0.16b
+; CHECK-GI-NEXT:    uaddlv h1, v1.16b
+; CHECK-GI-NEXT:    mov w0, v0.s[0]
+; CHECK-GI-NEXT:    mov w2, v1.s[0]
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <2 x i128> @llvm.ctpop(<2 x i128> %d)
   ret <2 x i128> %s
 }
 
 define <3 x i128> @v3i128(<3 x i128> %d) {
-; CHECK-LABEL: v3i128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov d0, x4
-; CHECK-NEXT:    fmov d1, x2
-; CHECK-NEXT:    fmov d2, x0
-; CHECK-NEXT:    mov v0.d[1], x5
-; CHECK-NEXT:    mov v1.d[1], x3
-; CHECK-NEXT:    mov v2.d[1], x1
-; CHECK-NEXT:    mov x1, xzr
-; CHECK-NEXT:    mov x3, xzr
-; CHECK-NEXT:    mov x5, xzr
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    cnt v2.16b, v2.16b
-; CHECK-NEXT:    addv b0, v0.16b
-; CHECK-NEXT:    addv b1, v1.16b
-; CHECK-NEXT:    addv b2, v2.16b
-; CHECK-NEXT:    fmov x0, d2
-; CHECK-NEXT:    fmov x2, d1
-; CHECK-NEXT:    fmov x4, d0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v3i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov d0, x4
+; CHECK-SD-NEXT:    fmov d1, x2
+; CHECK-SD-NEXT:    fmov d2, x0
+; CHECK-SD-NEXT:    mov v0.d[1], x5
+; CHECK-SD-NEXT:    mov v1.d[1], x3
+; CHECK-SD-NEXT:    mov v2.d[1], x1
+; CHECK-SD-NEXT:    mov x1, xzr
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    mov x5, xzr
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    cnt v1.16b, v1.16b
+; CHECK-SD-NEXT:    cnt v2.16b, v2.16b
+; CHECK-SD-NEXT:    addv b0, v0.16b
+; CHECK-SD-NEXT:    addv b1, v1.16b
+; CHECK-SD-NEXT:    addv b2, v2.16b
+; CHECK-SD-NEXT:    fmov x0, d2
+; CHECK-SD-NEXT:    fmov x2, d1
+; CHECK-SD-NEXT:    fmov x4, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v0.d[0], x0
+; CHECK-GI-NEXT:    mov v1.d[0], x2
+; CHECK-GI-NEXT:    mov v2.d[0], x4
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    mov v1.d[1], x3
+; CHECK-GI-NEXT:    mov v2.d[1], x5
+; CHECK-GI-NEXT:    mov x1, xzr
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    mov x5, xzr
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v1.16b, v1.16b
+; CHECK-GI-NEXT:    cnt v2.16b, v2.16b
+; CHECK-GI-NEXT:    uaddlv h0, v0.16b
+; CHECK-GI-NEXT:    uaddlv h1, v1.16b
+; CHECK-GI-NEXT:    uaddlv h2, v2.16b
+; CHECK-GI-NEXT:    mov w0, v0.s[0]
+; CHECK-GI-NEXT:    mov w2, v1.s[0]
+; CHECK-GI-NEXT:    mov w4, v2.s[0]
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <3 x i128> @llvm.ctpop(<3 x i128> %d)
   ret <3 x i128> %s
 }
 
 define <4 x i128> @v4i128(<4 x i128> %d) {
-; CHECK-LABEL: v4i128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov d0, x6
-; CHECK-NEXT:    fmov d1, x4
-; CHECK-NEXT:    fmov d2, x2
-; CHECK-NEXT:    fmov d3, x0
-; CHECK-NEXT:    mov v1.d[1], x5
-; CHECK-NEXT:    mov v2.d[1], x3
-; CHECK-NEXT:    mov v0.d[1], x7
-; CHECK-NEXT:    mov v3.d[1], x1
-; CHECK-NEXT:    mov x1, xzr
-; CHECK-NEXT:    mov x3, xzr
-; CHECK-NEXT:    mov x5, xzr
-; CHECK-NEXT:    mov x7, xzr
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    cnt v2.16b, v2.16b
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v3.16b, v3.16b
-; CHECK-NEXT:    addv b1, v1.16b
-; CHECK-NEXT:    addv b2, v2.16b
-; CHECK-NEXT:    addv b0, v0.16b
-; CHECK-NEXT:    addv b3, v3.16b
-; CHECK-NEXT:    fmov x2, d2
-; CHECK-NEXT:    fmov x4, d1
-; CHECK-NEXT:    fmov x6, d0
-; CHECK-NEXT:    fmov x0, d3
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v4i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov d0, x6
+; CHECK-SD-NEXT:    fmov d1, x4
+; CHECK-SD-NEXT:    fmov d2, x2
+; CHECK-SD-NEXT:    fmov d3, x0
+; CHECK-SD-NEXT:    mov v1.d[1], x5
+; CHECK-SD-NEXT:    mov v2.d[1], x3
+; CHECK-SD-NEXT:    mov v0.d[1], x7
+; CHECK-SD-NEXT:    mov v3.d[1], x1
+; CHECK-SD-NEXT:    mov x1, xzr
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    mov x5, xzr
+; CHECK-SD-NEXT:    mov x7, xzr
+; CHECK-SD-NEXT:    cnt v1.16b, v1.16b
+; CHECK-SD-NEXT:    cnt v2.16b, v2.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    cnt v3.16b, v3.16b
+; CHECK-SD-NEXT:    addv b1, v1.16b
+; CHECK-SD-NEXT:    addv b2, v2.16b
+; CHECK-SD-NEXT:    addv b0, v0.16b
+; CHECK-SD-NEXT:    addv b3, v3.16b
+; CHECK-SD-NEXT:    fmov x2, d2
+; CHECK-SD-NEXT:    fmov x4, d1
+; CHECK-SD-NEXT:    fmov x6, d0
+; CHECK-SD-NEXT:    fmov x0, d3
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v0.d[0], x0
+; CHECK-GI-NEXT:    mov v1.d[0], x2
+; CHECK-GI-NEXT:    mov v2.d[0], x4
+; CHECK-GI-NEXT:    mov v3.d[0], x6
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    mov v1.d[1], x3
+; CHECK-GI-NEXT:    mov v2.d[1], x5
+; CHECK-GI-NEXT:    mov v3.d[1], x7
+; CHECK-GI-NEXT:    mov x1, xzr
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    mov x5, xzr
+; CHECK-GI-NEXT:    mov x7, xzr
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v1.16b, v1.16b
+; CHECK-GI-NEXT:    cnt v2.16b, v2.16b
+; CHECK-GI-NEXT:    cnt v3.16b, v3.16b
+; CHECK-GI-NEXT:    uaddlv h0, v0.16b
+; CHECK-GI-NEXT:    uaddlv h1, v1.16b
+; CHECK-GI-NEXT:    uaddlv h2, v2.16b
+; CHECK-GI-NEXT:    uaddlv h3, v3.16b
+; CHECK-GI-NEXT:    mov w0, v0.s[0]
+; CHECK-GI-NEXT:    mov w2, v1.s[0]
+; CHECK-GI-NEXT:    mov w4, v2.s[0]
+; CHECK-GI-NEXT:    mov w6, v3.s[0]
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <4 x i128> @llvm.ctpop(<4 x i128> %d)
   ret <4 x i128> %s
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-GI: {{.*}}
-; CHECK-SD: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/cttz.ll b/llvm/test/CodeGen/AArch64/cttz.ll
index 9bc0970deeeda..41843e03cb81e 100644
--- a/llvm/test/CodeGen/AArch64/cttz.ll
+++ b/llvm/test/CodeGen/AArch64/cttz.ll
@@ -1,40 +1,42 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
-; CHECK-GI:       warning: Instruction selection used fallback path for v2i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v32i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v2i16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v16i16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v8i32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v2i128
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i128
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i128
 
 define void @v2i8(ptr %p1) {
-; CHECK-LABEL: v2i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ld1 { v0.b }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    movi v1.2s, #1
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-NEXT:    orr v0.2s, #1, lsl #8
-; CHECK-NEXT:    sub v1.2s, v0.2s, v1.2s
-; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    movi v1.2s, #32
-; CHECK-NEXT:    clz v0.2s, v0.2s
-; CHECK-NEXT:    sub v0.2s, v1.2s, v0.2s
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    strb w9, [x0]
-; CHECK-NEXT:    strb w8, [x0, #1]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v2i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1 { v0.b }[0], [x0]
+; CHECK-SD-NEXT:    add x8, x0, #1
+; CHECK-SD-NEXT:    movi v1.2s, #1
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-SD-NEXT:    orr v0.2s, #1, lsl #8
+; CHECK-SD-NEXT:    sub v1.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT:    movi v1.2s, #32
+; CHECK-SD-NEXT:    clz v0.2s, v0.2s
+; CHECK-SD-NEXT:    sub v0.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    strb w9, [x0]
+; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ld1 { v1.b }[0], [x0]
+; CHECK-GI-NEXT:    ldr b2, [x0, #1]
+; CHECK-GI-NEXT:    movi d0, #0xffffffffffffffff
+; CHECK-GI-NEXT:    mov v1.s[1], v2.s[0]
+; CHECK-GI-NEXT:    add v0.2s, v1.2s, v0.2s
+; CHECK-GI-NEXT:    bic v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mov w8, v0.s[1]
+; CHECK-GI-NEXT:    mov v0.b[1], w8
+; CHECK-GI-NEXT:    add x8, x0, #1
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    st1 { v0.b }[0], [x0]
+; CHECK-GI-NEXT:    st1 { v0.b }[1], [x8]
+; CHECK-GI-NEXT:    ret
 entry:
   %d = load <2 x i8>, ptr %p1
   %s = call <2 x i8> @llvm.cttz(<2 x i8> %d, i1 false)
@@ -43,27 +45,51 @@ entry:
 }
 
 define void @v3i8(ptr %p1) {
-; CHECK-LABEL: v3i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    movi v1.4h, #1
-; CHECK-NEXT:    zip1 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT:    orr v0.4h, #1, lsl #8
-; CHECK-NEXT:    sub v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    movi v1.4h, #16
-; CHECK-NEXT:    clz v0.4h, v0.4h
-; CHECK-NEXT:    sub v0.4h, v1.4h, v0.4h
-; CHECK-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
-; CHECK-NEXT:    umov w8, v0.h[2]
-; CHECK-NEXT:    str s1, [sp, #12]
-; CHECK-NEXT:    ldrh w9, [sp, #12]
-; CHECK-NEXT:    strb w8, [x0, #2]
-; CHECK-NEXT:    strh w9, [x0]
-; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v3i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sub sp, sp, #16
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    movi v1.4h, #1
+; CHECK-SD-NEXT:    zip1 v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    orr v0.4h, #1, lsl #8
+; CHECK-SD-NEXT:    sub v1.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT:    movi v1.4h, #16
+; CHECK-SD-NEXT:    clz v0.4h, v0.4h
+; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    str s1, [sp, #12]
+; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
+; CHECK-SD-NEXT:    strb w8, [x0, #2]
+; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    add sp, sp, #16
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w9, [x0]
+; CHECK-GI-NEXT:    mov w8, #65535 // =0xffff
+; CHECK-GI-NEXT:    ldrb w10, [x0, #1]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    ldrb w9, [x0, #2]
+; CHECK-GI-NEXT:    mov v0.h[1], w8
+; CHECK-GI-NEXT:    mov v1.h[1], w10
+; CHECK-GI-NEXT:    mov v0.h[2], w8
+; CHECK-GI-NEXT:    add x8, x0, #1
+; CHECK-GI-NEXT:    mov v1.h[2], w9
+; CHECK-GI-NEXT:    add x9, x0, #2
+; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    and v0.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    st1 { v0.b }[0], [x0]
+; CHECK-GI-NEXT:    st1 { v0.b }[1], [x8]
+; CHECK-GI-NEXT:    st1 { v0.b }[2], [x9]
+; CHECK-GI-NEXT:    ret
 entry:
   %d = load <3 x i8>, ptr %p1
   %s = call <3 x i8> @llvm.cttz(<3 x i8> %d, i1 false)
@@ -72,20 +98,47 @@ entry:
 }
 
 define void @v4i8(ptr %p1) {
-; CHECK-LABEL: v4i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    movi v1.4h, #1
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    orr v0.4h, #1, lsl #8
-; CHECK-NEXT:    sub v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    movi v1.4h, #16
-; CHECK-NEXT:    clz v0.4h, v0.4h
-; CHECK-NEXT:    sub v0.4h, v1.4h, v0.4h
-; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT:    str s0, [x0]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    movi v1.4h, #1
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    orr v0.4h, #1, lsl #8
+; CHECK-SD-NEXT:    sub v1.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT:    movi v1.4h, #16
+; CHECK-SD-NEXT:    clz v0.4h, v0.4h
+; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-SD-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w9, [x0]
+; CHECK-GI-NEXT:    mov w8, #255 // =0xff
+; CHECK-GI-NEXT:    fmov s0, w9
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v0.h[1], w9
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    mov v0.h[2], w9
+; CHECK-GI-NEXT:    mov v1.h[2], w8
+; CHECK-GI-NEXT:    fmov w9, s3
+; CHECK-GI-NEXT:    mov v0.h[3], w9
+; CHECK-GI-NEXT:    mov v1.h[3], w8
+; CHECK-GI-NEXT:    eor v2.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    and v0.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    str w8, [x0]
+; CHECK-GI-NEXT:    ret
 entry:
   %d = load <4 x i8>, ptr %p1
   %s = call <4 x i8> @llvm.cttz(<4 x i8> %d, i1 false)
@@ -136,38 +189,68 @@ entry:
 }
 
 define <32 x i8> @v32i8(<32 x i8> %d) {
-; CHECK-LABEL: v32i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v2.16b, #1
-; CHECK-NEXT:    sub v3.16b, v0.16b, v2.16b
-; CHECK-NEXT:    sub v2.16b, v1.16b, v2.16b
-; CHECK-NEXT:    bic v0.16b, v3.16b, v0.16b
-; CHECK-NEXT:    bic v1.16b, v2.16b, v1.16b
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v32i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v2.16b, #1
+; CHECK-SD-NEXT:    sub v3.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT:    sub v2.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-SD-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    cnt v1.16b, v1.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v32i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v2.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v3.16b, v0.16b, v2.16b
+; CHECK-GI-NEXT:    add v2.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-GI-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v1.16b, v1.16b
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <32 x i8> @llvm.cttz(<32 x i8> %d, i1 false)
   ret <32 x i8> %s
 }
 
 define void @v2i16(ptr %p1) {
-; CHECK-LABEL: v2i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ld1 { v0.h }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #2
-; CHECK-NEXT:    movi v1.2s, #1
-; CHECK-NEXT:    ld1 { v0.h }[2], [x8]
-; CHECK-NEXT:    orr v0.2s, #1, lsl #16
-; CHECK-NEXT:    sub v1.2s, v0.2s, v1.2s
-; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    movi v1.2s, #32
-; CHECK-NEXT:    clz v0.2s, v0.2s
-; CHECK-NEXT:    sub v0.2s, v1.2s, v0.2s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    str h0, [x0]
-; CHECK-NEXT:    str h1, [x0, #2]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ld1 { v0.h }[0], [x0]
+; CHECK-SD-NEXT:    add x8, x0, #2
+; CHECK-SD-NEXT:    movi v1.2s, #1
+; CHECK-SD-NEXT:    ld1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    orr v0.2s, #1, lsl #16
+; CHECK-SD-NEXT:    sub v1.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT:    movi v1.2s, #32
+; CHECK-SD-NEXT:    clz v0.2s, v0.2s
+; CHECK-SD-NEXT:    sub v0.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    mov s1, v0.s[1]
+; CHECK-SD-NEXT:    str h0, [x0]
+; CHECK-SD-NEXT:    str h1, [x0, #2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #65535 // =0xffff
+; CHECK-GI-NEXT:    ld1 { v1.h }[0], [x0]
+; CHECK-GI-NEXT:    ldr h2, [x0, #2]
+; CHECK-GI-NEXT:    mov v0.s[0], w8
+; CHECK-GI-NEXT:    mov v1.s[1], v2.s[0]
+; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    add x8, x0, #2
+; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    add v0.2s, v1.2s, v0.2s
+; CHECK-GI-NEXT:    and v0.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-GI-NEXT:    str h0, [x0]
+; CHECK-GI-NEXT:    st1 { v0.h }[1], [x8]
+; CHECK-GI-NEXT:    ret
 entry:
   %d = load <2 x i16>, ptr %p1
   %s = call <2 x i16> @llvm.cttz(<2 x i16> %d, i1 false)
@@ -176,19 +259,40 @@ entry:
 }
 
 define void @v3i16(ptr %p1) {
-; CHECK-LABEL: v3i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.4h, #1
-; CHECK-NEXT:    ldr d1, [x0]
-; CHECK-NEXT:    add x8, x0, #4
-; CHECK-NEXT:    sub v0.4h, v1.4h, v0.4h
-; CHECK-NEXT:    bic v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    movi v1.4h, #16
-; CHECK-NEXT:    clz v0.4h, v0.4h
-; CHECK-NEXT:    sub v0.4h, v1.4h, v0.4h
-; CHECK-NEXT:    st1 { v0.h }[2], [x8]
-; CHECK-NEXT:    str s0, [x0]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v3i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.4h, #1
+; CHECK-SD-NEXT:    ldr d1, [x0]
+; CHECK-SD-NEXT:    add x8, x0, #4
+; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-SD-NEXT:    bic v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    movi v1.4h, #16
+; CHECK-SD-NEXT:    clz v0.4h, v0.4h
+; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #65535 // =0xffff
+; CHECK-GI-NEXT:    ldr h1, [x0]
+; CHECK-GI-NEXT:    add x9, x0, #2
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    add x10, x0, #4
+; CHECK-GI-NEXT:    ld1 { v1.h }[1], [x9]
+; CHECK-GI-NEXT:    mov v0.h[1], w8
+; CHECK-GI-NEXT:    ld1 { v1.h }[2], [x10]
+; CHECK-GI-NEXT:    mov v0.h[2], w8
+; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    and v0.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-GI-NEXT:    str h0, [x0]
+; CHECK-GI-NEXT:    st1 { v0.h }[1], [x9]
+; CHECK-GI-NEXT:    st1 { v0.h }[2], [x10]
+; CHECK-GI-NEXT:    ret
 entry:
   %d = load <3 x i16>, ptr %p1
   %s = call <3 x i16> @llvm.cttz(<3 x i16> %d, i1 false)
@@ -245,19 +349,32 @@ entry:
 }
 
 define <16 x i16> @v16i16(<16 x i16> %d) {
-; CHECK-LABEL: v16i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v2.8h, #1
-; CHECK-NEXT:    sub v3.8h, v0.8h, v2.8h
-; CHECK-NEXT:    sub v2.8h, v1.8h, v2.8h
-; CHECK-NEXT:    bic v0.16b, v3.16b, v0.16b
-; CHECK-NEXT:    bic v1.16b, v2.16b, v1.16b
-; CHECK-NEXT:    movi v2.8h, #16
-; CHECK-NEXT:    clz v0.8h, v0.8h
-; CHECK-NEXT:    clz v1.8h, v1.8h
-; CHECK-NEXT:    sub v0.8h, v2.8h, v0.8h
-; CHECK-NEXT:    sub v1.8h, v2.8h, v1.8h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v16i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v2.8h, #1
+; CHECK-SD-NEXT:    sub v3.8h, v0.8h, v2.8h
+; CHECK-SD-NEXT:    sub v2.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-SD-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-SD-NEXT:    movi v2.8h, #16
+; CHECK-SD-NEXT:    clz v0.8h, v0.8h
+; CHECK-SD-NEXT:    clz v1.8h, v1.8h
+; CHECK-SD-NEXT:    sub v0.8h, v2.8h, v0.8h
+; CHECK-SD-NEXT:    sub v1.8h, v2.8h, v1.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v16i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v2.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v3.8h, v0.8h, v2.8h
+; CHECK-GI-NEXT:    add v2.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-GI-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v1.16b, v1.16b
+; CHECK-GI-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-GI-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <16 x i16> @llvm.cttz(<16 x i16> %d, i1 false)
   ret <16 x i16> %s
@@ -289,15 +406,29 @@ entry:
 }
 
 define <3 x i32> @v3i32(<3 x i32> %d) {
-; CHECK-LABEL: v3i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    sub v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    movi v1.4s, #32
-; CHECK-NEXT:    clz v0.4s, v0.4s
-; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.4s, #1
+; CHECK-SD-NEXT:    sub v1.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    movi v1.4s, #32
+; CHECK-SD-NEXT:    clz v0.4s, v0.4s
+; CHECK-SD-NEXT:    sub v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    eor v2.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    and v0.16b, v2.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-GI-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <3 x i32> @llvm.cttz(<3 x i32> %d, i1 false)
   ret <3 x i32> %s
@@ -329,19 +460,34 @@ entry:
 }
 
 define <8 x i32> @v8i32(<8 x i32> %d) {
-; CHECK-LABEL: v8i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v2.4s, #1
-; CHECK-NEXT:    sub v3.4s, v0.4s, v2.4s
-; CHECK-NEXT:    sub v2.4s, v1.4s, v2.4s
-; CHECK-NEXT:    bic v0.16b, v3.16b, v0.16b
-; CHECK-NEXT:    bic v1.16b, v2.16b, v1.16b
-; CHECK-NEXT:    movi v2.4s, #32
-; CHECK-NEXT:    clz v0.4s, v0.4s
-; CHECK-NEXT:    clz v1.4s, v1.4s
-; CHECK-NEXT:    sub v0.4s, v2.4s, v0.4s
-; CHECK-NEXT:    sub v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v2.4s, #1
+; CHECK-SD-NEXT:    sub v3.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    sub v2.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-SD-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-SD-NEXT:    movi v2.4s, #32
+; CHECK-SD-NEXT:    clz v0.4s, v0.4s
+; CHECK-SD-NEXT:    clz v1.4s, v1.4s
+; CHECK-SD-NEXT:    sub v0.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT:    sub v1.4s, v2.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v2.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v3.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    add v2.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-GI-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v1.16b, v1.16b
+; CHECK-GI-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-GI-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-GI-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-GI-NEXT:    uaddlp v1.4s, v1.8h
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <8 x i32> @llvm.cttz(<8 x i32> %d, i1 false)
   ret <8 x i32> %s
@@ -376,154 +522,305 @@ entry:
 }
 
 define <3 x i64> @v3i64(<3 x i64> %d) {
-; CHECK-LABEL: v3i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    dup v1.2d, x8
-; CHECK-NEXT:    sub v3.2d, v0.2d, v1.2d
-; CHECK-NEXT:    sub v1.2d, v2.2d, v1.2d
-; CHECK-NEXT:    bic v0.16b, v3.16b, v0.16b
-; CHECK-NEXT:    bic v1.16b, v1.16b, v2.16b
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v1.8h, v1.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    uaddlp v2.4s, v1.8h
-; CHECK-NEXT:    uaddlp v0.2d, v0.4s
-; CHECK-NEXT:    uaddlp v2.2d, v2.4s
-; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    mov w8, #1 // =0x1
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    dup v1.2d, x8
+; CHECK-SD-NEXT:    sub v3.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    sub v1.2d, v2.2d, v1.2d
+; CHECK-SD-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-SD-NEXT:    bic v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    cnt v1.16b, v1.16b
+; CHECK-SD-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-SD-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-SD-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-SD-NEXT:    uaddlp v2.4s, v1.8h
+; CHECK-SD-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-SD-NEXT:    uaddlp v2.2d, v2.4s
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    mov v3.16b, v0.16b
+; CHECK-GI-NEXT:    movi v4.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    fmov x9, d2
+; CHECK-GI-NEXT:    mov v3.d[1], v1.d[0]
+; CHECK-GI-NEXT:    sub x8, x8, #1
+; CHECK-GI-NEXT:    bic x8, x8, x9
+; CHECK-GI-NEXT:    add v0.2d, v0.2d, v4.2d
+; CHECK-GI-NEXT:    fmov d1, x8
+; CHECK-GI-NEXT:    bic v0.16b, v0.16b, v3.16b
+; CHECK-GI-NEXT:    cnt v1.8b, v1.8b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    uaddlv h1, v1.8b
+; CHECK-GI-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-GI-NEXT:    mov w8, v1.s[0]
+; CHECK-GI-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <3 x i64> @llvm.cttz(<3 x i64> %d, i1 false)
   ret <3 x i64> %s
 }
 
 define <4 x i64> @v4i64(<4 x i64> %d) {
-; CHECK-LABEL: v4i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    dup v2.2d, x8
-; CHECK-NEXT:    sub v3.2d, v0.2d, v2.2d
-; CHECK-NEXT:    sub v2.2d, v1.2d, v2.2d
-; CHECK-NEXT:    bic v0.16b, v3.16b, v0.16b
-; CHECK-NEXT:    bic v1.16b, v2.16b, v1.16b
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v1.8h, v1.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    uaddlp v1.4s, v1.8h
-; CHECK-NEXT:    uaddlp v0.2d, v0.4s
-; CHECK-NEXT:    uaddlp v1.2d, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #1 // =0x1
+; CHECK-SD-NEXT:    dup v2.2d, x8
+; CHECK-SD-NEXT:    sub v3.2d, v0.2d, v2.2d
+; CHECK-SD-NEXT:    sub v2.2d, v1.2d, v2.2d
+; CHECK-SD-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-SD-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    cnt v1.16b, v1.16b
+; CHECK-SD-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-SD-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-SD-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-SD-NEXT:    uaddlp v1.4s, v1.8h
+; CHECK-SD-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-SD-NEXT:    uaddlp v1.2d, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v2.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v3.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    add v2.2d, v1.2d, v2.2d
+; CHECK-GI-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-GI-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v1.16b, v1.16b
+; CHECK-GI-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-GI-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-GI-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-GI-NEXT:    uaddlp v1.4s, v1.8h
+; CHECK-GI-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-GI-NEXT:    uaddlp v1.2d, v1.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <4 x i64> @llvm.cttz(<4 x i64> %d, i1 false)
   ret <4 x i64> %s
 }
 
 define <2 x i128> @v2i128(<2 x i128> %d) {
-; CHECK-LABEL: v2i128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rbit x8, x1
-; CHECK-NEXT:    rbit x9, x0
-; CHECK-NEXT:    rbit x10, x3
-; CHECK-NEXT:    rbit x11, x2
-; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    mov x1, xzr
-; CHECK-NEXT:    clz x8, x8
-; CHECK-NEXT:    clz x9, x9
-; CHECK-NEXT:    clz x10, x10
-; CHECK-NEXT:    add x8, x8, #64
-; CHECK-NEXT:    mov x3, xzr
-; CHECK-NEXT:    csel x0, x9, x8, ne
-; CHECK-NEXT:    clz x8, x11
-; CHECK-NEXT:    add x9, x10, #64
-; CHECK-NEXT:    cmp x2, #0
-; CHECK-NEXT:    csel x2, x8, x9, ne
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v2i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rbit x8, x1
+; CHECK-SD-NEXT:    rbit x9, x0
+; CHECK-SD-NEXT:    rbit x10, x3
+; CHECK-SD-NEXT:    rbit x11, x2
+; CHECK-SD-NEXT:    cmp x0, #0
+; CHECK-SD-NEXT:    mov x1, xzr
+; CHECK-SD-NEXT:    clz x8, x8
+; CHECK-SD-NEXT:    clz x9, x9
+; CHECK-SD-NEXT:    clz x10, x10
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    csel x0, x9, x8, ne
+; CHECK-SD-NEXT:    clz x8, x11
+; CHECK-SD-NEXT:    add x9, x10, #64
+; CHECK-SD-NEXT:    cmp x2, #0
+; CHECK-SD-NEXT:    csel x2, x8, x9, ne
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #-1 // =0xffffffffffffffff
+; CHECK-GI-NEXT:    subs x9, x0, #1
+; CHECK-GI-NEXT:    adc x10, x1, x8
+; CHECK-GI-NEXT:    subs x11, x2, #1
+; CHECK-GI-NEXT:    bic x9, x9, x0
+; CHECK-GI-NEXT:    mov v0.d[0], x9
+; CHECK-GI-NEXT:    bic x9, x11, x2
+; CHECK-GI-NEXT:    adc x8, x3, x8
+; CHECK-GI-NEXT:    mov v1.d[0], x9
+; CHECK-GI-NEXT:    bic x9, x10, x1
+; CHECK-GI-NEXT:    bic x8, x8, x3
+; CHECK-GI-NEXT:    mov x1, xzr
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v1.16b, v1.16b
+; CHECK-GI-NEXT:    uaddlv h0, v0.16b
+; CHECK-GI-NEXT:    uaddlv h1, v1.16b
+; CHECK-GI-NEXT:    mov w0, v0.s[0]
+; CHECK-GI-NEXT:    mov w2, v1.s[0]
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <2 x i128> @llvm.cttz(<2 x i128> %d, i1 false)
   ret <2 x i128> %s
 }
 
 define <3 x i128> @v3i128(<3 x i128> %d) {
-; CHECK-LABEL: v3i128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rbit x8, x1
-; CHECK-NEXT:    rbit x9, x0
-; CHECK-NEXT:    rbit x11, x3
-; CHECK-NEXT:    rbit x10, x2
-; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    rbit x12, x5
-; CHECK-NEXT:    clz x8, x8
-; CHECK-NEXT:    clz x9, x9
-; CHECK-NEXT:    clz x11, x11
-; CHECK-NEXT:    add x8, x8, #64
-; CHECK-NEXT:    clz x10, x10
-; CHECK-NEXT:    mov x1, xzr
-; CHECK-NEXT:    csel x0, x9, x8, ne
-; CHECK-NEXT:    add x8, x11, #64
-; CHECK-NEXT:    cmp x2, #0
-; CHECK-NEXT:    rbit x9, x4
-; CHECK-NEXT:    csel x2, x10, x8, ne
-; CHECK-NEXT:    clz x8, x12
-; CHECK-NEXT:    add x8, x8, #64
-; CHECK-NEXT:    cmp x4, #0
-; CHECK-NEXT:    mov x3, xzr
-; CHECK-NEXT:    clz x9, x9
-; CHECK-NEXT:    mov x5, xzr
-; CHECK-NEXT:    csel x4, x9, x8, ne
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v3i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rbit x8, x1
+; CHECK-SD-NEXT:    rbit x9, x0
+; CHECK-SD-NEXT:    rbit x11, x3
+; CHECK-SD-NEXT:    rbit x10, x2
+; CHECK-SD-NEXT:    cmp x0, #0
+; CHECK-SD-NEXT:    rbit x12, x5
+; CHECK-SD-NEXT:    clz x8, x8
+; CHECK-SD-NEXT:    clz x9, x9
+; CHECK-SD-NEXT:    clz x11, x11
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    clz x10, x10
+; CHECK-SD-NEXT:    mov x1, xzr
+; CHECK-SD-NEXT:    csel x0, x9, x8, ne
+; CHECK-SD-NEXT:    add x8, x11, #64
+; CHECK-SD-NEXT:    cmp x2, #0
+; CHECK-SD-NEXT:    rbit x9, x4
+; CHECK-SD-NEXT:    csel x2, x10, x8, ne
+; CHECK-SD-NEXT:    clz x8, x12
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    cmp x4, #0
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    clz x9, x9
+; CHECK-SD-NEXT:    mov x5, xzr
+; CHECK-SD-NEXT:    csel x4, x9, x8, ne
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #-1 // =0xffffffffffffffff
+; CHECK-GI-NEXT:    subs x9, x0, #1
+; CHECK-GI-NEXT:    adc x10, x1, x8
+; CHECK-GI-NEXT:    subs x11, x2, #1
+; CHECK-GI-NEXT:    bic x9, x9, x0
+; CHECK-GI-NEXT:    adc x12, x3, x8
+; CHECK-GI-NEXT:    subs x13, x4, #1
+; CHECK-GI-NEXT:    bic x11, x11, x2
+; CHECK-GI-NEXT:    mov v0.d[0], x9
+; CHECK-GI-NEXT:    bic x9, x13, x4
+; CHECK-GI-NEXT:    mov v1.d[0], x11
+; CHECK-GI-NEXT:    mov v2.d[0], x9
+; CHECK-GI-NEXT:    adc x8, x5, x8
+; CHECK-GI-NEXT:    bic x9, x10, x1
+; CHECK-GI-NEXT:    bic x10, x12, x3
+; CHECK-GI-NEXT:    bic x8, x8, x5
+; CHECK-GI-NEXT:    mov x1, xzr
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    mov x5, xzr
+; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    mov v1.d[1], x10
+; CHECK-GI-NEXT:    mov v2.d[1], x8
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v1.16b, v1.16b
+; CHECK-GI-NEXT:    cnt v2.16b, v2.16b
+; CHECK-GI-NEXT:    uaddlv h0, v0.16b
+; CHECK-GI-NEXT:    uaddlv h1, v1.16b
+; CHECK-GI-NEXT:    uaddlv h2, v2.16b
+; CHECK-GI-NEXT:    mov w0, v0.s[0]
+; CHECK-GI-NEXT:    mov w2, v1.s[0]
+; CHECK-GI-NEXT:    mov w4, v2.s[0]
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <3 x i128> @llvm.cttz(<3 x i128> %d, i1 false)
   ret <3 x i128> %s
 }
 
 define <4 x i128> @v4i128(<4 x i128> %d) {
-; CHECK-LABEL: v4i128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rbit x9, x1
-; CHECK-NEXT:    rbit x10, x0
-; CHECK-NEXT:    rbit x8, x3
-; CHECK-NEXT:    rbit x11, x2
-; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    mov x1, xzr
-; CHECK-NEXT:    clz x9, x9
-; CHECK-NEXT:    clz x10, x10
-; CHECK-NEXT:    clz x8, x8
-; CHECK-NEXT:    add x9, x9, #64
-; CHECK-NEXT:    add x8, x8, #64
-; CHECK-NEXT:    mov x3, xzr
-; CHECK-NEXT:    csel x0, x10, x9, ne
-; CHECK-NEXT:    clz x9, x11
-; CHECK-NEXT:    rbit x10, x4
-; CHECK-NEXT:    rbit x11, x5
-; CHECK-NEXT:    cmp x2, #0
-; CHECK-NEXT:    mov x5, xzr
-; CHECK-NEXT:    csel x2, x9, x8, ne
-; CHECK-NEXT:    clz x8, x10
-; CHECK-NEXT:    rbit x10, x7
-; CHECK-NEXT:    clz x9, x11
-; CHECK-NEXT:    cmp x4, #0
-; CHECK-NEXT:    rbit x11, x6
-; CHECK-NEXT:    add x9, x9, #64
-; CHECK-NEXT:    mov x7, xzr
-; CHECK-NEXT:    csel x4, x8, x9, ne
-; CHECK-NEXT:    clz x8, x10
-; CHECK-NEXT:    clz x9, x11
-; CHECK-NEXT:    add x8, x8, #64
-; CHECK-NEXT:    cmp x6, #0
-; CHECK-NEXT:    csel x6, x9, x8, ne
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: v4i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rbit x9, x1
+; CHECK-SD-NEXT:    rbit x10, x0
+; CHECK-SD-NEXT:    rbit x8, x3
+; CHECK-SD-NEXT:    rbit x11, x2
+; CHECK-SD-NEXT:    cmp x0, #0
+; CHECK-SD-NEXT:    mov x1, xzr
+; CHECK-SD-NEXT:    clz x9, x9
+; CHECK-SD-NEXT:    clz x10, x10
+; CHECK-SD-NEXT:    clz x8, x8
+; CHECK-SD-NEXT:    add x9, x9, #64
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    csel x0, x10, x9, ne
+; CHECK-SD-NEXT:    clz x9, x11
+; CHECK-SD-NEXT:    rbit x10, x4
+; CHECK-SD-NEXT:    rbit x11, x5
+; CHECK-SD-NEXT:    cmp x2, #0
+; CHECK-SD-NEXT:    mov x5, xzr
+; CHECK-SD-NEXT:    csel x2, x9, x8, ne
+; CHECK-SD-NEXT:    clz x8, x10
+; CHECK-SD-NEXT:    rbit x10, x7
+; CHECK-SD-NEXT:    clz x9, x11
+; CHECK-SD-NEXT:    cmp x4, #0
+; CHECK-SD-NEXT:    rbit x11, x6
+; CHECK-SD-NEXT:    add x9, x9, #64
+; CHECK-SD-NEXT:    mov x7, xzr
+; CHECK-SD-NEXT:    csel x4, x8, x9, ne
+; CHECK-SD-NEXT:    clz x8, x10
+; CHECK-SD-NEXT:    clz x9, x11
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    cmp x6, #0
+; CHECK-SD-NEXT:    csel x6, x9, x8, ne
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #-1 // =0xffffffffffffffff
+; CHECK-GI-NEXT:    subs x9, x0, #1
+; CHECK-GI-NEXT:    adc x10, x1, x8
+; CHECK-GI-NEXT:    subs x11, x2, #1
+; CHECK-GI-NEXT:    bic x9, x9, x0
+; CHECK-GI-NEXT:    adc x12, x3, x8
+; CHECK-GI-NEXT:    subs x13, x4, #1
+; CHECK-GI-NEXT:    mov v0.d[0], x9
+; CHECK-GI-NEXT:    adc x14, x5, x8
+; CHECK-GI-NEXT:    subs x15, x6, #1
+; CHECK-GI-NEXT:    bic x9, x11, x2
+; CHECK-GI-NEXT:    bic x11, x13, x4
+; CHECK-GI-NEXT:    mov v1.d[0], x9
+; CHECK-GI-NEXT:    bic x9, x15, x6
+; CHECK-GI-NEXT:    mov v2.d[0], x11
+; CHECK-GI-NEXT:    mov v3.d[0], x9
+; CHECK-GI-NEXT:    adc x8, x7, x8
+; CHECK-GI-NEXT:    bic x9, x10, x1
+; CHECK-GI-NEXT:    bic x10, x14, x5
+; CHECK-GI-NEXT:    bic x8, x8, x7
+; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    bic x9, x12, x3
+; CHECK-GI-NEXT:    mov x1, xzr
+; CHECK-GI-NEXT:    mov v1.d[1], x9
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    mov x5, xzr
+; CHECK-GI-NEXT:    mov v2.d[1], x10
+; CHECK-GI-NEXT:    mov v3.d[1], x8
+; CHECK-GI-NEXT:    mov x7, xzr
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v1.16b, v1.16b
+; CHECK-GI-NEXT:    cnt v2.16b, v2.16b
+; CHECK-GI-NEXT:    cnt v3.16b, v3.16b
+; CHECK-GI-NEXT:    uaddlv h0, v0.16b
+; CHECK-GI-NEXT:    uaddlv h1, v1.16b
+; CHECK-GI-NEXT:    uaddlv h2, v2.16b
+; CHECK-GI-NEXT:    uaddlv h3, v3.16b
+; CHECK-GI-NEXT:    mov w0, v0.s[0]
+; CHECK-GI-NEXT:    mov w2, v1.s[0]
+; CHECK-GI-NEXT:    mov w4, v2.s[0]
+; CHECK-GI-NEXT:    mov w6, v3.s[0]
+; CHECK-GI-NEXT:    ret
 entry:
   %s = call <4 x i128> @llvm.cttz(<4 x i128> %d, i1 false)
   ret <4 x i128> %s
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}